Bump version v0.8.0

Fix crash when no --files provided; use microphone input instead

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: [3.9, '3.10', 3.11, 3.12]
     steps:
       - uses: actions/checkout@v2
       
@@ -25,7 +25,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Cache Python dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/pip
@@ -35,7 +35,7 @@ jobs:
             ${{ runner.os }}-pip-${{ matrix.python-version }}-
       
       - name: Install system dependencies
-        run: sudo apt-get update && sudo apt-get install -y ffmpeg portaudio19-dev
+        run: sudo apt-get update && sudo apt-get install -y portaudio19-dev
       
       - name: Install Python dependencies
         run: |
@@ -52,7 +52,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: [3.8, 3.9, '3.10', 3.11]
+        python-version: [3.9, '3.10', 3.11, 3.12]
 
     steps:
       - uses: actions/checkout@v2
@@ -77,7 +77,7 @@ jobs:
   build-and-push-docker-cpu:
     needs: [run-tests, check-code-format]
     runs-on: ubuntu-22.04
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
     steps:
       - uses: actions/checkout@v2
       
@@ -103,7 +103,7 @@ jobs:
     needs: [run-tests, check-code-format, build-and-push-docker-cpu]
     timeout-minutes: 20
     runs-on: ubuntu-22.04
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
     steps:
       - uses: actions/checkout@v2
 
@@ -128,6 +128,35 @@ jobs:
           push: true
           tags: ghcr.io/collabora/whisperlive-gpu:latest
 
+  build-and-push-docker-openvino:
+    needs: [run-tests, check-code-format, build-and-push-docker-cpu]
+    timeout-minutes: 20
+    runs-on: ubuntu-22.04
+    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Docker Prune
+        run: docker system prune -af
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push Docker GPU image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: docker/Dockerfile.openvino
+          push: true
+          tags: ghcr.io/collabora/whisperlive-openvino:latest
+
   publish-to-pypi:
     needs: [run-tests, check-code-format]
     runs-on: ubuntu-22.04
@@ -135,23 +164,23 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Cache Python dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cache/pip
             !~/.cache/pip/log
-          key: ubuntu-latest-pip-3.8-${{ hashFiles('requirements/server.txt', 'requirements/client.txt') }}
+          key: ubuntu-latest-pip-3.9-${{ hashFiles('requirements/server.txt', 'requirements/client.txt') }}
           restore-keys: |
-            ubuntu-latest-pip-3.8-
+            ubuntu-latest-pip-3.9-
 
       - name: Install system dependencies
-        run: sudo apt-get update && sudo apt-get install -y ffmpeg portaudio19-dev
+        run: sudo apt-get update && sudo apt-get install -y portaudio19-dev
 
       - name: Install Python dependencies
         run: |

Audio-Transcription-Chrome/README.md

@@ -27,6 +27,7 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to
 When using the Audio Transcription extension, you have the following options:
  - **Use Collabora Server**: We provide a demo server which runs the whisper small model.
  - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper.
+ - **Download SRT file at Stop Capture**: Select if you want to download the srt file for the session at stop capture.
  - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English.
  - **Model Size**: Select the whisper model size to run the server with.
 

Audio-Transcription-Chrome/audiopreprocessor.js

@@ -0,0 +1,77 @@
+class AudioPreProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.sampleRate = sampleRate || 48000;
+    this.targetSampleRate = 16000;
+    this.inputSamplesNeeded = this.sampleRate * 0.5; // 0.5s
+    this.inputBuffer = new Float32Array(this.inputSamplesNeeded);
+    this.inputWriteOffset = 0;
+    this.processCount = 0;
+    this.audioDetectedCount = 0;
+  }
+
+  process(inputs, outputs) {
+    this.processCount++;
+    
+    const input = inputs[0];
+    const output = outputs[0];
+   
+    if (!input || input.length === 0) {
+      return true;
+    }
+
+    for (let channel = 0; channel < Math.min(input.length, output.length); channel++) {
+      if (input[channel] && output[channel]) {
+        output[channel].set(input[channel]);
+      }
+    }
+
+    let monoInput;
+    if (input.length === 1) {
+      monoInput = input[0];
+    } else if (input.length >= 2) {
+      monoInput = new Float32Array(input[0].length);
+      for (let i = 0; i < input[0].length; i++) {
+        monoInput[i] = (input[0][i] + (input[1] ? input[1][i] : 0)) * 0.5;
+      }
+    } else {
+      return true;
+    }
+
+    if (!monoInput || monoInput.length === 0) {
+      return true;
+    }
+
+    let inputOffset = 0;
+    while (inputOffset < monoInput.length) {
+      const remainingBuffer = this.inputSamplesNeeded - this.inputWriteOffset;
+      const toCopy = Math.min(remainingBuffer, monoInput.length - inputOffset);
+      this.inputBuffer.set(monoInput.subarray(inputOffset, inputOffset + toCopy), this.inputWriteOffset);
+
+      this.inputWriteOffset += toCopy;
+      inputOffset += toCopy;
+
+      if (this.inputWriteOffset === this.inputSamplesNeeded) {
+        const downsampled = this.downsampleTo16kHz(this.inputBuffer);
+        this.port.postMessage(downsampled);
+
+        this.inputWriteOffset = 0;
+      }
+    }
+
+    return true;
+  }
+
+  downsampleTo16kHz(inputBuffer) {
+    const ratio = this.sampleRate / this.targetSampleRate;
+    const length = Math.floor(inputBuffer.length / ratio);
+    const result = new Float32Array(length);
+    for (let i = 0; i < length; i++) {
+      const idx = Math.floor(i * ratio);
+      result[i] = inputBuffer[idx];
+    }
+    return result;
+  }
+}
+
+registerProcessor('audiopreprocessor', AudioPreProcessor);

Audio-Transcription-Chrome/background.js

@@ -159,6 +159,7 @@ async function startCapture(options) {
           task: options.task,
           modelSize: options.modelSize,
           useVad: options.useVad,
+          saveCaptions: options.saveCaptions,
         },
       });
     } else {
@@ -174,14 +175,14 @@ async function startCapture(options) {
  * Stops the capture process and performs cleanup.
  * @returns {Promise<void>} - A Promise that resolves when the capture process is stopped successfully.
  */
-async function stopCapture() {
+async function stopCapture(options) {
   const optionTabId = await getLocalStorageValue("optionTabId");
   const currentTabId = await getLocalStorageValue("currentTabId");
 
   if (optionTabId) {
     res = await sendMessageToTab(currentTabId, {
       type: "STOP",
-      data: { currentTabId: currentTabId },
+      data: { currentTabId: currentTabId, saveCaptions: options.saveCaptions },
     });
     await removeChromeTab(optionTabId);
   }
@@ -196,7 +197,7 @@ chrome.runtime.onMessage.addListener(async (message) => {
   if (message.action === "startCapture") {
     startCapture(message);
   } else if (message.action === "stopCapture") {
-    stopCapture();
+    stopCapture(message);
   } else if (message.action === "updateSelectedLanguage") {
     const detectedLanguage = message.detectedLanguage;
     chrome.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage });
@@ -204,7 +205,7 @@ chrome.runtime.onMessage.addListener(async (message) => {
   } else if (message.action === "toggleCaptureButtons") {
     chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false });
     chrome.storage.local.set({ capturingState: { isCapturing: false } })
-    stopCapture();
+    stopCapture({saveCaptions: message.saveCaptions});
   }
 });
 

Audio-Transcription-Chrome/content.js

@@ -1,10 +1,45 @@
-
-
 var elem_container = null;
 var elem_text = null;
 
 var segments = [];
 var text_segments = [];
+var allSegments = [];
+var lastIncompleteSegment = null;
+
+function formatTime(seconds) {
+  const date = new Date(seconds * 1000);
+  const hh = String(date.getUTCHours()).padStart(2, '0');
+  const mm = String(date.getUTCMinutes()).padStart(2, '0');
+  const ss = String(date.getUTCSeconds()).padStart(2, '0');
+  const mmm = String(date.getUTCMilliseconds()).padStart(3, '0');
+  return `${hh}:${mm}:${ss},${mmm}`;
+}
+
+function generateSRT() {
+    return allSegments
+    .map((seg, i) => {
+    const start = formatTime(seg.start);
+    const end   = formatTime(seg.end);
+    const text  = seg.text.trim().replace(/[\r\n]+/g, ' ');
+    return `${i + 1}\n${start} --> ${end}\n${text}`;
+    })
+    .join('\n\n');
+}
+
+function downloadSRT() {
+    console.log("downloadSRT called");
+    console.log("Total segments for SRT:", allSegments.length);
+    const srtBlob = new Blob([generateSRT()], { type: 'text/srt;charset=utf-8' });
+    const url = URL.createObjectURL(srtBlob);
+    const a = document.createElement('a');
+    a.href = url;
+    a.download = 'captions.srt';
+    a.style.display = 'none';
+    document.body.appendChild(a);
+    a.click();
+    URL.revokeObjectURL(url);
+    document.body.removeChild(a);
+}
 
 function initPopupElement() {
   if (document.getElementById('popupElement')) {
@@ -32,7 +67,7 @@ function initPopupElement() {
   closePopupButton.style.cursor = 'pointer';
   closePopupButton.addEventListener('click', async () => {
     popupContainer.style.display = 'none';
-    await browser.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false });
+    await chrome.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false });
   });
   buttonContainer.appendChild(closePopupButton);
   popupContainer.appendChild(buttonContainer);
@@ -169,8 +204,25 @@ function remove_element() {
 
 chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
     const { type, data } = request;
-    
-    if (type === "STOP") {
+    const saveCaptions = data.saveCaptions;
+
+    if (type === "STOP") {        
+        if (saveCaptions === true) {
+            // If there is a last incomplete segment, push it to allSegments
+            if (lastIncompleteSegment && lastIncompleteSegment.text && lastIncompleteSegment.text.trim() !== "") {
+                // Apply same Python logic: check if transcript is empty OR start >= last end
+                if (allSegments.length === 0 || parseFloat(lastIncompleteSegment.start) >= parseFloat(allSegments[allSegments.length - 1].end)) {
+                    allSegments.push({
+                        start: lastIncompleteSegment.start,
+                        end: lastIncompleteSegment.end,
+                        text: lastIncompleteSegment.text
+                    });
+                    console.log("Added final incomplete segment");
+                }
+            }
+        
+            downloadSRT();
+        }
         remove_element();
         sendResponse({data: "STOPPED"});
         return true;
@@ -184,53 +236,77 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
 
     init_element();
 
-    message = JSON.parse(data);
-    message = message["segments"];
-
-    var text = '';
-    for (var i = 0; i < message.length; i++) {
-        text += message[i].text + ' ';
-    }
-    text = text.replace(/(\r\n|\n|\r)/gm, "");
-    
-    var elem = document.getElementById('t3');
-    elem.innerHTML = text;
-
-    var line_height_style = getStyle('t3', 'line-height');
-    var line_height = parseInt(line_height_style.substring(0, line_height_style.length - 2));
-    var divHeight = elem.offsetHeight;
-    var lines = divHeight / line_height;
-
-    text_segments = [];
-    text_segments = get_lines(elem, line_height);
-    
-    elem.innerHTML = '';
-
-    if (text_segments.length > 2) {
-        for (var i = 0; i < 3; i++) {
-            document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
+    try {
+        const message = JSON.parse(data.data);
+        const segments = message["segments"];
+        
+        if (saveCaptions === true) {
+            segments.forEach(seg => {
+                if (seg.completed === true && 
+                    (allSegments.length === 0 || parseFloat(seg.start) >= parseFloat(allSegments[allSegments.length - 1].end))) {
+                    allSegments.push({
+                        start: seg.start,
+                        end: seg.end,
+                        text: seg.text
+                    });
+                    
+                    lastIncompleteSegment = null;
+                } else if (seg.completed !== true) {
+                    lastIncompleteSegment = seg;
+                }
+            });
         }
-    } else {
-        for (var i = 0; i < 3; i++) {
-            document.getElementById('t' + i).innerHTML = '';
+        var text = '';
+        for (var i = 0; i < segments.length; i++) {
+            text += segments[i].text + ' ';
         }
-    }
+        text = text.replace(/(\r\n|\n|\r)/gm, "");
+        
+        var elem = document.getElementById('t3');
+        if (elem) {
+            elem.innerHTML = text;
 
-    if (text_segments.length <= 2) {
-        for (var i = 0; i < text_segments.length; i++) {
-            document.getElementById('t' + i).innerHTML = text_segments[i];
-        }
-    } else {
-        for (var i = 0; i < 3; i++) {
-            document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
-        }
-    }
+            var line_height_style = getStyle('t3', 'line-height');
+            var line_height = parseInt(line_height_style.substring(0, line_height_style.length - 2));
+            var divHeight = elem.offsetHeight;
+            var lines = divHeight / line_height;
 
-    for (var i = 1; i < 3; i++)
-    {
-        var parent_elem = document.getElementById('t' + (i - 1));
-        var elem = document.getElementById('t' + i);
-        elem.style.top = parent_elem.offsetHeight + parent_elem.offsetTop + 'px';
+            text_segments = [];
+            text_segments = get_lines(elem, line_height);
+            
+            elem.innerHTML = '';
+
+            if (text_segments.length > 2) {
+                for (var i = 0; i < 3; i++) {
+                    document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
+                }
+            } else {
+                for (var i = 0; i < 3; i++) {
+                    document.getElementById('t' + i).innerHTML = '';
+                }
+            }
+
+            if (text_segments.length <= 2) {
+                for (var i = 0; i < text_segments.length; i++) {
+                    document.getElementById('t' + i).innerHTML = text_segments[i];
+                }
+            } else {
+                for (var i = 0; i < 3; i++) {
+                    document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
+                }
+            }
+
+            for (var i = 1; i < 3; i++)
+            {
+                var parent_elem = document.getElementById('t' + (i - 1));
+                var elem = document.getElementById('t' + i);
+                if (parent_elem && elem) {
+                    elem.style.top = parent_elem.offsetHeight + parent_elem.offsetTop + 'px';
+                }
+            }
+        }
+    } catch (error) {
+        console.error("Error processing message:", error);
     }
 
     sendResponse({});

Audio-Transcription-Chrome/manifest.json

@@ -1,14 +1,20 @@
-  {
+{
     "manifest_version": 3,
 
     "name": "Audio Transcription",
     "version": "1.0.0",
     "description": "This extension captures the audio on the current tab, sends it to a server for transcription and shows the transcription in Real-time.",
-    
+
     "options_page": "options.html",
     "background": {
         "service_worker": "background.js"
     },
+    "web_accessible_resources": [
+        {
+        "resources": ["audiopreprocessor.js"],
+        "matches": ["<all_urls>"]
+        }
+    ],
     "permissions": [
         "storage",
         "activeTab",

Audio-Transcription-Chrome/options.js

@@ -31,41 +31,6 @@ function sendMessageToTab(tabId, data) {
   });
 }
 
-
-/**
- * Resamples the audio data to a target sample rate of 16kHz.
- * @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data.
- * @param {number} [origSampleRate=44100] - The original sample rate of the audio data.
- * @returns {Float32Array} The resampled audio data at 16kHz.
- */
-function resampleTo16kHZ(audioData, origSampleRate = 44100) {
-  // Convert the audio data to a Float32Array
-  const data = new Float32Array(audioData);
-
-  // Calculate the desired length of the resampled data
-  const targetLength = Math.round(data.length * (16000 / origSampleRate));
-
-  // Create a new Float32Array for the resampled data
-  const resampledData = new Float32Array(targetLength);
-
-  // Calculate the spring factor and initialize the first and last values
-  const springFactor = (data.length - 1) / (targetLength - 1);
-  resampledData[0] = data[0];
-  resampledData[targetLength - 1] = data[data.length - 1];
-
-  // Resample the audio data
-  for (let i = 1; i < targetLength - 1; i++) {
-    const index = i * springFactor;
-    const leftIndex = Math.floor(index).toFixed();
-    const rightIndex = Math.ceil(index).toFixed();
-    const fraction = index - leftIndex;
-    resampledData[i] = data[leftIndex] + (data[rightIndex] - data[leftIndex]) * fraction;
-  }
-
-  // Return the resampled data
-  return resampledData;
-}
-
 function generateUUID() {
   let dt = new Date().getTime();
   const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
@@ -76,24 +41,99 @@ function generateUUID() {
   return uuid;
 }
 
+// Global variables for audio processing
+let audioContext = null;
+let preNode = null;
+let socket = null;
+let isServerReady = false;
+let currentStream = null;
+let currentOptions = null;
+
+// AudioWorklet URL - make sure this path matches your manifest.json
+const WORKLET_URL = chrome.runtime.getURL('audiopreprocessor.js');
+
+async function initAudioWorklet(stream) {
+  audioContext = new AudioContext();
+  if (audioContext.state === 'suspended') {
+    await audioContext.resume();
+  }
+
+  try {
+    await audioContext.audioWorklet.addModule(WORKLET_URL);
+    preNode = new AudioWorkletNode(audioContext, 'audiopreprocessor');
+    const mediaStream = audioContext.createMediaStreamSource(stream);
+    
+    mediaStream.connect(preNode);
+    preNode.connect(audioContext.destination);
+    preNode.port.onmessage = (event) => {
+      const data = event.data;
+      
+      
+      const audio16k = data; // Float32Array @ 16 kHz
+      
+      if (socket && socket.readyState === WebSocket.OPEN && isServerReady) {
+        socket.send(audio16k);
+      }
+    };
+        
+    // Test if we can hear audio (this will help verify the audio path)
+    
+  } catch (error) {
+    console.error("Error initializing AudioWorklet:", error);
+    throw error;
+  }
+}
+
+function cleanupAudio() {
+  
+  if (preNode) {
+    preNode.port.onmessage = null;
+    preNode.disconnect();
+    preNode = null;
+  }
+  
+  if (audioContext) {
+    audioContext.close();
+    audioContext = null;
+  }
+
+  if (currentStream) {
+    currentStream.getTracks().forEach(track => {
+      track.stop();
+      console.log("Stopped track:", track.kind);
+    });
+    currentStream = null;
+  }
+}
 
 /**
  * Starts recording audio from the captured tab.
  * @param {Object} option - The options object containing the currentTabId.
  */
 async function startRecord(option) {
+  currentOptions = option;
   const stream = await captureTabAudio();
   const uuid = generateUUID();
 
   if (stream) {
-    // call when the stream inactive
+    currentStream = stream;
     stream.oninactive = () => {
+      cleanupAudio();
       window.close();
     };
-    const socket = new WebSocket(`ws://${option.host}:${option.port}/`);
-    let isServerReady = false;
+
+    try {
+      await initAudioWorklet(stream);
+    } catch (error) {
+      console.error("Failed to initialize AudioWorklet:", error);
+      return;
+    }
+
+    socket = new WebSocket(`ws://${option.host}:${option.port}/`);
+    isServerReady = false;
     let language = option.language;
-    socket.onopen = function(e) { 
+
+    socket.onopen = function(e) {
       socket.send(
         JSON.stringify({
           uid: uuid,
@@ -129,7 +169,6 @@ async function startRecord(option) {
         language = data["language"];
         
         // send message to popup.js to update dropdown
-        // console.log(language);
         chrome.runtime.sendMessage({
           action: "updateSelectedLanguage",
           detectedLanguage: language,
@@ -139,43 +178,33 @@ async function startRecord(option) {
       }
 
       if (data["message"] === "DISCONNECT"){
-        chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false })        
+        chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false, saveCaptions: option.saveCaptions });        
         return;
       }
 
-      res = await sendMessageToTab(option.currentTabId, {
+      const res = await sendMessageToTab(option.currentTabId, {
         type: "transcript",
-        data: event.data,
+        data: {
+          data: event.data,
+          saveCaptions: option.saveCaptions,
+        },
       });
     };
 
-    
-    const audioDataCache = [];
-    const context = new AudioContext();
-    const mediaStream = context.createMediaStreamSource(stream);
-    const recorder = context.createScriptProcessor(4096, 1, 1);
-
-    recorder.onaudioprocess = async (event) => {
-      if (!context || !isServerReady) return;
-
-      const inputData = event.inputBuffer.getChannelData(0);
-      const audioData16kHz = resampleTo16kHZ(inputData, context.sampleRate);
-
-      audioDataCache.push(inputData);
-
-      socket.send(audioData16kHz);
+    socket.onclose = () => {
+      cleanupAudio();
+    };
+
+    socket.onerror = (error) => {
+      cleanupAudio();
     };
 
-    // Prevent page mute
-    mediaStream.connect(recorder);
-    recorder.connect(context.destination);
-    mediaStream.connect(context.destination);
-    // }
   } else {
     window.close();
   }
 }
 
+
 /**
  * Listener for incoming messages from the extension's background script.
  * @param {Object} request - The message request object.

Audio-Transcription-Chrome/popup.html

@@ -19,6 +19,10 @@
     <input type="checkbox" id="useVadCheckbox">
     <label for="useVadCheckbox">Use Voice Activity Detection</label>
   </div>
+  <div class="checkbox-container">
+    <input type="checkbox" id="saveCaptionsCheckbox">
+    <label for="saveCaptions">Download SRT file at Stop Capture</label>
+  </div>
   <div class="dropdown-container">
     <label for="languageDropdown">Select Language:</label>
     <select id="languageDropdown">

Audio-Transcription-Chrome/popup.js

@@ -5,6 +5,7 @@ document.addEventListener("DOMContentLoaded", function () {
 
   const useServerCheckbox = document.getElementById("useServerCheckbox");
   const useVadCheckbox = document.getElementById("useVadCheckbox");
+  const saveCaptionsCheckbox = document.getElementById("saveCaptionsCheckbox");
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
   const modelSizeDropdown = document.getElementById('modelSizeDropdown');
@@ -38,6 +39,12 @@ document.addEventListener("DOMContentLoaded", function () {
     }
   });
 
+  chrome.storage.local.get("saveCaptionsState", ({ saveCaptionsState }) => {
+    if (saveCaptionsState !== undefined) {
+      saveCaptionsCheckbox.checked = saveCaptionsState;
+    }
+  });
+
   chrome.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => {
     if (storedLanguage !== undefined) {
       languageDropdown.value = storedLanguage;
@@ -88,6 +95,7 @@ document.addEventListener("DOMContentLoaded", function () {
         task: selectedTask,
         modelSize: selectedModelSize,
         useVad: useVadCheckbox.checked,
+        saveCaptions: saveCaptionsCheckbox.checked,
       }, () => {
         // Update capturing state in storage and toggle the buttons
         chrome.storage.local.set({ capturingState: { isCapturing: true } }, () => {
@@ -105,7 +113,11 @@ document.addEventListener("DOMContentLoaded", function () {
     }
 
     // Send a message to the background script to stop capturing
-    chrome.runtime.sendMessage({ action: "stopCapture" }, () => {
+    chrome.runtime.sendMessage(
+      { 
+        action: "stopCapture",
+        saveCaptions: saveCaptionsCheckbox.checked,
+      }, () => {
       // Update capturing state in storage and toggle the buttons
       chrome.storage.local.set({ capturingState: { isCapturing: false } }, () => {
         toggleCaptureButtons(false);
@@ -128,6 +140,7 @@ document.addEventListener("DOMContentLoaded", function () {
     stopButton.disabled = !isCapturing;
     useServerCheckbox.disabled = isCapturing;
     useVadCheckbox.disabled = isCapturing;
+    saveCaptionsCheckbox.disabled = isCapturing;
     modelSizeDropdown.disabled = isCapturing;
     languageDropdown.disabled = isCapturing;
     taskDropdown.disabled = isCapturing; 
@@ -146,6 +159,11 @@ document.addEventListener("DOMContentLoaded", function () {
     chrome.storage.local.set({ useVadState });
   });
 
+  saveCaptionsCheckbox.addEventListener("change", () => {
+    const saveCaptionsState = saveCaptionsCheckbox.checked;
+    chrome.storage.local.set({ saveCaptionsState });
+  });
+
   languageDropdown.addEventListener('change', function() {
     if (languageDropdown.value === "") {
       selectedLanguage = null;

Audio-Transcription-Firefox/README.md

@@ -25,6 +25,7 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to
 When using the Audio Transcription extension, you have the following options:
  - **Use Collabora Server**: We provide a demo server which runs the whisper small model.
  - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper.
+ - **Download SRT file at Stop Capture**: Select if you want to download the srt file for the session at stop capture.
  - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English.
   - **Model Size**: Select the whisper model size to run the server with.
 

Audio-Transcription-Firefox/audiopreprocessor.js

@@ -0,0 +1,70 @@
+class AudioPreProcessor extends AudioWorkletProcessor {
+    constructor() {
+      super();
+      this.sampleRate = sampleRate || 48000;
+      this.targetSampleRate = 16000;
+      this.inputSamplesNeeded = this.sampleRate * 0.5;
+      this.inputBuffer = new Float32Array(this.inputSamplesNeeded);
+      this.inputWriteOffset = 0;
+    }
+  
+    process(inputs, outputs) {
+      const input = inputs[0]; 
+      const output = outputs[0];
+      if (!input || input.length === 0) {
+        return true;
+      }
+      for (let channel = 0; channel < Math.min(input.length, output.length); channel++) {
+        if (input[channel] && output[channel]) {
+          output[channel].set(input[channel]);
+        }
+      }
+
+      let monoInput;
+      if (input.length === 1) {
+        monoInput = input[0];
+      } else if (input.length > 1) {
+        monoInput = new Float32Array(input[0].length);
+        for (let channel = 0; channel < input.length; channel++) {
+          monoInput.set(input[channel], 0);
+        }
+      }
+
+      if (!monoInput) {
+        return true;
+      }
+  
+      let inputOffset = 0;
+      while (inputOffset < monoInput.length) {
+        const remainingBuffer = this.inputSamplesNeeded - this.inputWriteOffset;
+        const toCopy = Math.min(remainingBuffer, monoInput.length - inputOffset);
+        this.inputBuffer.set(monoInput.subarray(inputOffset, inputOffset + toCopy), this.inputWriteOffset);
+
+        this.inputWriteOffset += toCopy;
+        inputOffset += toCopy;
+
+        if (this.inputWriteOffset === this.inputSamplesNeeded) {
+          const downsampled = this.downsampleTo16kHz(this.inputBuffer);
+          this.port.postMessage(downsampled);
+
+          this.inputWriteOffset = 0;
+        }
+      }
+
+      return true;
+    }
+  
+    downsampleTo16kHz(inputBuffer) {
+      const ratio = this.sampleRate / this.targetSampleRate;
+      const length = Math.floor(inputBuffer.length / ratio); 
+      const result = new Float32Array(length);
+      for (let i = 0; i < length; i++) {
+        const idx = Math.floor(i * ratio);
+        result[i] = inputBuffer[idx];
+      }
+      return result;
+    }
+  }
+  
+  registerProcessor('audiopreprocessor', AudioPreProcessor);
+  

Audio-Transcription-Firefox/content.js

@@ -1,144 +1,162 @@
 let socket = null;
 let isCapturing = false;
-let mediaStream = null;
 let audioContext = null;
-let scriptProcessor = null;
 let language = null;
-
 let isPaused = false;
+let preNode = null;
+let allSegments = [];
+let lastIncompleteSegment = null;
 
-const mediaElements = document.querySelectorAll('video, audio');
-mediaElements.forEach((mediaElement) => {
-  mediaElement.addEventListener('play', handlePlaybackStateChange);
-  mediaElement.addEventListener('pause', handlePlaybackStateChange);
-});
-
-
-function handlePlaybackStateChange(event) {
-  isPaused = event.target.paused;
+function formatTime(seconds) {
+  const date = new Date(seconds * 1000);
+  const hh = String(date.getUTCHours()).padStart(2, '0');
+  const mm = String(date.getUTCMinutes()).padStart(2, '0');
+  const ss = String(date.getUTCSeconds()).padStart(2, '0');
+  const mmm = String(date.getUTCMilliseconds()).padStart(3, '0');
+  return `${hh}:${mm}:${ss},${mmm}`;
 }
 
+function generateSRT() {
+    return allSegments
+    .map((seg, i) => {
+    const start = formatTime(seg.start);
+    const end   = formatTime(seg.end);
+    const text  = seg.text.trim().replace(/[\r\n]+/g, ' ');
+    return `${i + 1}\n${start} --> ${end}\n${text}`;
+    })
+    .join('\n\n');
+}
+
+function downloadSRT() {
+    const srtBlob = new Blob([generateSRT()], { type: 'text/srt;charset=utf-8' });
+    const url = URL.createObjectURL(srtBlob);
+    const a = document.createElement('a');
+    a.href = url;
+    a.download = 'captions.srt';
+    a.style.display = 'none';
+    document.body.appendChild(a);
+    a.click();
+    URL.revokeObjectURL(url);
+    document.body.removeChild(a);
+}
+
+
 function generateUUID() {
   let dt = new Date().getTime();
-  const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
+  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {
     const r = (dt + Math.random() * 16) % 16 | 0;
     dt = Math.floor(dt / 16);
     return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
   });
-  return uuid;
 }
 
 
-/**
- * Resamples the audio data to a target sample rate of 16kHz.
- * @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data.
- * @param {number} [origSampleRate=44100] - The original sample rate of the audio data.
- * @returns {Float32Array} The resampled audio data at 16kHz.
- */
-function resampleTo16kHZ(audioData, origSampleRate = 44100) {
-  // Convert the audio data to a Float32Array
-  const data = new Float32Array(audioData);
+document.querySelectorAll('video, audio').forEach(el => {
+  el.addEventListener('play', () => { isPaused = false; });
+  el.addEventListener('pause', () => { isPaused = true; });
+});
 
-  // Calculate the desired length of the resampled data
-  const targetLength = Math.round(data.length * (16000 / origSampleRate));
 
-  // Create a new Float32Array for the resampled data
-  const resampledData = new Float32Array(targetLength);
+function setupMessageHandler() {
+  if (preNode) {
+    preNode.port.onmessage = e => {
+      const audio16k = e.data; 
+      if (isCapturing && socket && socket.readyState === WebSocket.OPEN && !isPaused) {
+        socket.send(audio16k);
+      }
+    };
+  }
+}
 
-  // Calculate the spring factor and initialize the first and last values
-  const springFactor = (data.length - 1) / (targetLength - 1);
-  resampledData[0] = data[0];
-  resampledData[targetLength - 1] = data[data.length - 1];
 
-  // Resample the audio data
-  for (let i = 1; i < targetLength - 1; i++) {
-    const index = i * springFactor;
-    const leftIndex = Math.floor(index).toFixed();
-    const rightIndex = Math.ceil(index).toFixed();
-    const fraction = index - leftIndex;
-    resampledData[i] = data[leftIndex] + (data[rightIndex] - data[leftIndex]) * fraction;
+const WORKLET_URL = browser.runtime.getURL('audiopreprocessor.js');
+
+async function initAudioWorklet() {
+  if (audioContext && preNode) {
+    setupMessageHandler();
+    return;
+  }
+  audioContext = new AudioContext();
+  await audioContext.audioWorklet.addModule(WORKLET_URL);
+
+  preNode = new AudioWorkletNode(audioContext, 'audiopreprocessor');
+  document.querySelectorAll('audio, video').forEach(el => {
+    let src;
+    try {
+      src = audioContext.createMediaElementSource(el);
+    } catch(e) {
+      console.warn('Could not create MediaElementSource for', el, e);
+      return;
+    }
+    src.connect(preNode);
+    src.connect(audioContext.destination);
+  });
+
+  preNode.connect(audioContext.destination);
+
+  setupMessageHandler();
+}
+
+async function startRecording(data) {
+  if (!audioContext) {
+    await initAudioWorklet();
   }
 
-  // Return the resampled data
-  return resampledData;
+  const uid = generateUUID();
+  socket = new WebSocket(`ws://${data.host}:${data.port}/`);
+  language = data.language;
+
+  socket.onopen = () => {
+    socket.send(JSON.stringify({
+      uid,
+      language: data.language,
+      task: data.task,
+      model: data.modelSize,
+      use_vad: data.useVad
+    }));
+  };
+
+  let serverReady = false;
+  socket.onmessage = async event => {
+    const msg = JSON.parse(event.data);
+    if (msg.uid !== uid) return;
+
+    if (msg.status === 'WAIT') {
+      await browser.runtime.sendMessage({ action: 'showPopup', data: msg.message });
+      return;
+    }
+    if (!serverReady && msg.message === 'SERVER_READY') {
+      serverReady = true;
+      return;
+    }
+    if (!language && msg.language) {
+      language = msg.language;
+      await browser.runtime.sendMessage({ action: 'updateSelectedLanguage', data: language });
+      return;
+    }
+    if (msg.message === 'DISCONNECT') {
+      await browser.runtime.sendMessage({ action: 'toggleCaptureButtons' });
+      return;
+    }
+    if (msg.segments) {
+      await browser.runtime.sendMessage({ action: 'transcript', data: {data: event.data, saveCaption: data.saveCaption} });
+    }
+  };
+
+  isCapturing = true;
 }
 
-function startRecording(data) {
-    socket = new WebSocket(`ws://${data.host}:${data.port}/`);
-    language = data.language;
+function stopRecording() {
+  isCapturing = false;
+  if (socket) {
+    socket.close();
+    socket = null;
+  }
 
-    const uuid = generateUUID();
-    socket.onopen = function(e) { 
-      socket.send(
-        JSON.stringify({
-            uid: uuid,
-            language: data.language,
-            task: data.task,
-            model: data.modelSize,
-            use_vad: data.useVad
-        })
-      );
-    };
-
-    let isServerReady = false;
-    socket.onmessage = async (event) => {
-      const data = JSON.parse(event.data);
-      if (data["uid"] !== uuid)
-        return;
-      
-      if (data["status"] === "WAIT"){
-        await browser.runtime.sendMessage({ action: "showPopup", data: data["message"] })
-        return;
-      }
-      
-      if (!isServerReady && data["message"] === "SERVER_READY"){
-        isServerReady = true;
-        return;
-      }
-
-      if (language === null ){
-        language = data["language"];
-        await browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language })      
-        return
-      }
-
-      if (data["message"] === "DISCONNECT"){
-        await browser.runtime.sendMessage({ action: "toggleCaptureButtons", data: false })        
-        return
-      }
-
-      await browser.runtime.sendMessage({ action: "transcript", data: event.data })
-          .catch(function(error) {
-            console.error("Error sending message:", error);
-          });
-    };
-
-  // Access the audio stream from the current tab
-  navigator.mediaDevices.getUserMedia({ audio: true })
-    .then(function(stream) {
-      // Create a new MediaRecorder instance
-      const audioDataCache = [];
-      audioContext = new AudioContext();
-      mediaStream = audioContext.createMediaStreamSource(stream);
-      recorder = audioContext.createScriptProcessor(4096, 1, 1);
-
-      recorder.onaudioprocess = async (event) => {
-        if (!audioContext || !isCapturing || !isServerReady || isPaused) return;
-
-        const inputData = event.inputBuffer.getChannelData(0);
-        const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate);
-
-        audioDataCache.push(inputData);
-        
-        socket.send(audioData16kHz);
-      };
-
-      // Prevent page mute
-      mediaStream.connect(recorder);
-      recorder.connect(audioContext.destination);
-    })
+  remove_element();
 }
 
+
 var elem_container = null;
 var elem_text = null;
 
@@ -308,6 +326,8 @@ function remove_element() {
 
 browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
   const { action, data } = request;
+  const saveCaption = data.saveCaption || false;
+
   if (action === "startCapture") {
       isCapturing = true;
       startRecording(data);
@@ -318,12 +338,20 @@ browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
         socket.close();
         socket = null;
     }
-    
-    if (audioContext) {
-        audioContext.close();
-        audioContext = null;
-        mediaStream = null;
-        recorder = null;
+
+
+    if (saveCaption === true) {
+      if (lastIncompleteSegment && lastIncompleteSegment.text && lastIncompleteSegment.text.trim() !== "") {
+          if (allSegments.length === 0 || parseFloat(lastIncompleteSegment.start) >= parseFloat(allSegments[allSegments.length - 1].end)) {
+              allSegments.push({
+                  start: lastIncompleteSegment.start,
+                  end: lastIncompleteSegment.end,
+                  text: lastIncompleteSegment.text
+              });
+          }
+      }
+  
+      downloadSRT();
     }
 
     remove_element();
@@ -337,8 +365,25 @@ browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
   } else if (action === "show_transcript"){
     if (!isCapturing) return;
     init_element();    
-    message = JSON.parse(data);
+    message = JSON.parse(data.data);
     message = message["segments"];
+
+    if (saveCaption === true) {
+      message.forEach(seg => {
+          if (seg.completed === true && 
+              (allSegments.length === 0 || parseFloat(seg.start) >= parseFloat(allSegments[allSegments.length - 1].end))) {
+              allSegments.push({
+                  start: seg.start,
+                  end: seg.end,
+                  text: seg.text
+              });
+              
+              lastIncompleteSegment = null;
+          } else if (seg.completed !== true) {
+              lastIncompleteSegment = seg;
+          }
+      });
+    }
     
     var text = '';
     for (var i = 0; i < message.length; i++) {

Audio-Transcription-Firefox/manifest.json

@@ -8,6 +8,9 @@
       "activeTab",
       "<all_urls>"
     ],
+    "web_accessible_resources": [
+      "audiopreprocessor.js"
+    ],
     "background": {
       "scripts": ["background.js"],
       "persistent": false

Audio-Transcription-Firefox/popup.html

@@ -19,6 +19,10 @@
     <input type="checkbox" id="useVadCheckbox">
     <label for="useVadCheckbox">Use Voice Activity Detection</label>
   </div>
+  <div class="checkbox-container">
+    <input type="checkbox" id="saveCaptionCheckbox">
+    <label for="saveCaption">Download SRT file at Stop Capture</label>
+  </div>
   <textarea id="waitTextBox" style="display: none;"></textarea>
   <div class="dropdown-container">
     <label for="languageDropdown">Select Language:</label>

Audio-Transcription-Firefox/popup.js

@@ -4,6 +4,7 @@ document.addEventListener("DOMContentLoaded", function() {
 
   const useServerCheckbox = document.getElementById("useServerCheckbox");
   const useVadCheckbox = document.getElementById("useVadCheckbox");
+  const saveCaptionCheckbox = document.getElementById("saveCaptionCheckbox");
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
   const modelSizeDropdown = document.getElementById('modelSizeDropdown');
@@ -41,6 +42,12 @@ document.addEventListener("DOMContentLoaded", function() {
     }
   });
 
+  browser.storage.local.get("saveCaptionState", ({ saveCaptionState }) => {
+    if (saveCaptionState !== undefined) {
+      saveCaptionCheckbox.checked = saveCaptionState;
+    }
+  });
+
   browser.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => {
     if (storedLanguage !== undefined) {
       languageDropdown.value = storedLanguage;
@@ -85,6 +92,7 @@ document.addEventListener("DOMContentLoaded", function() {
               task: selectedTask,
               modelSize: selectedModelSize,
               useVad: useVadCheckbox.checked,
+              saveCaption: saveCaptionCheckbox.checked,
             } 
           });
         toggleCaptureButtons(true);
@@ -101,7 +109,7 @@ document.addEventListener("DOMContentLoaded", function() {
   stopButton.addEventListener("click", function() {
     browser.tabs.query({ active: true, currentWindow: true })
       .then(function(tabs) {
-        browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture" })
+        browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture", data: {saveCaption: saveCaptionCheckbox.checked, } })
           .then(function(response) {
             toggleCaptureButtons(false);
             browser.storage.local.set({ capturingState: { isCapturing: false } })
@@ -124,6 +132,7 @@ document.addEventListener("DOMContentLoaded", function() {
     stopButton.disabled = !isCapturing;
     useServerCheckbox.disabled = isCapturing;
     useVadCheckbox.disabled = isCapturing;
+    saveCaptionCheckbox.disabled = isCapturing;
     modelSizeDropdown.disabled = isCapturing;
     languageDropdown.disabled = isCapturing;
     taskDropdown.disabled = isCapturing; 
@@ -142,6 +151,11 @@ document.addEventListener("DOMContentLoaded", function() {
     browser.storage.local.set({ useVadState });
   });
 
+  saveCaptionCheckbox.addEventListener("change", () => {
+    const saveCaptionState = saveCaptionCheckbox.checked;
+    browser.storage.local.set({ saveCaptionState });
+  });
+
   languageDropdown.addEventListener('change', function() {
     if (languageDropdown.value === "") {
       selectedLanguage = null;

Audio-Transcription-iOS/AudioStream.swift

@@ -0,0 +1,229 @@
+//  AudioStream.swift
+//  Lecture2Quiz
+//
+//  Created by ParkMazorika on 4/27/25.
+//
+
+import AVFoundation
+
+/// Streams audio input to a WebSocket after converting and normalizing.
+class AudioStreamer {
+    private let engine = AVAudioEngine()
+    private let inputNode: AVAudioInputNode
+    private var inputFormat: AVAudioFormat?
+    private var isPaused: Bool = false
+    private var audioWebSocket: AudioWebSocket?
+    private var partialBuffer = Data()
+    private var isStreaming: Bool = false
+
+    private var bufferSize: AVAudioFrameCount = 1600  // ~100ms of audio
+    private var sampleRate: Double = 16000
+    private var channels: UInt32 = 1
+
+    private var converter: AVAudioConverter?
+
+    init(webSocket: AudioWebSocket) {
+        self.inputNode = engine.inputNode
+        self.audioWebSocket = webSocket
+
+        let inputFormat = inputNode.outputFormat(forBus: 0)
+        print("Input format: \(inputFormat)")
+
+        let outputFormat = AVAudioFormat(
+            commonFormat: .pcmFormatInt16,
+            sampleRate: 16000,
+            channels: 1,
+            interleaved: true
+        )!
+
+        self.converter = AVAudioConverter(from: inputFormat, to: outputFormat)
+        self.inputFormat = outputFormat
+    }
+
+    /// Configures the audio session for recording.
+    func configureAudioSession() {
+        let session = AVAudioSession.sharedInstance()
+        do {
+            try session.setCategory(.playAndRecord, mode: .default, options: [.allowBluetooth, .defaultToSpeaker])
+            try session.setPreferredSampleRate(48000)
+            try session.setPreferredInputNumberOfChannels(1)
+            try session.setMode(.videoChat)
+            try session.setActive(true, options: .notifyOthersOnDeactivation)
+            sampleRate = session.sampleRate
+            channels = UInt32(session.inputNumberOfChannels)
+            print("Sample rate: \(sampleRate)")
+            print("Input channels: \(channels)")
+        } catch {
+            print("Failed to configure audio session: \(error.localizedDescription)")
+        }
+    }
+
+    /// Starts capturing and streaming audio data.
+    func startStreaming() {
+        guard !isStreaming else {
+            print("Already streaming.")
+            return
+        }
+
+        configureAudioSession()
+
+        let format = AVAudioFormat(
+            commonFormat: .pcmFormatFloat32,
+            sampleRate: 48000,
+            channels: channels,
+            interleaved: true
+        )
+
+        guard let hardwareFormat = format else {
+            print("Failed to create audio format.")
+            return
+        }
+
+        self.inputFormat = hardwareFormat
+
+        inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: hardwareFormat) { [weak self] buffer, _ in
+            self?.processAudioBuffer(buffer)
+        }
+
+        do {
+            try engine.start()
+            isStreaming = true
+            print("AVAudioEngine started.")
+        } catch {
+            print("Failed to start AVAudioEngine: \(error.localizedDescription)")
+        }
+    }
+
+    /// Converts and sends the audio buffer to the server via WebSocket.
+    func processAudioBuffer(_ buffer: AVAudioPCMBuffer) {
+        guard let converter = self.converter else {
+            print("Audio converter is nil.")
+            return
+        }
+
+        if let floatChannelData = buffer.floatChannelData {
+            let frameLength = Int(buffer.frameLength)
+            let channelData = Array(UnsafeBufferPointer(start: floatChannelData.pointee, count: frameLength))
+            let rms = sqrt(channelData.map { $0 * $0 }.reduce(0, +) / Float(frameLength))
+            print("Audio RMS: \(rms)")
+            if rms < 0.001 {
+                print("Warning: Input volume is too low.")
+            }
+        }
+
+        let outputFormat = AVAudioFormat(
+            commonFormat: .pcmFormatInt16,
+            sampleRate: 16000,
+            channels: 1,
+            interleaved: true
+        )!
+
+        guard let newBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat, frameCapacity: 1600) else {
+            print("Failed to allocate PCM buffer.")
+            return
+        }
+
+        let inputBlock: AVAudioConverterInputBlock = { _, outStatus in
+            outStatus.pointee = .haveData
+            return buffer
+        }
+
+        var error: NSError?
+        converter.convert(to: newBuffer, error: &error, withInputFrom: inputBlock)
+
+        if let error = error {
+            print("Audio conversion failed: \(error.localizedDescription)")
+            return
+        }
+
+        print("Converted buffer frameLength: \(newBuffer.frameLength), sampleRate: \(newBuffer.format.sampleRate)")
+
+        if let audioData = convertToFloat32BytesLikePython(newBuffer) {
+            var completeData = partialBuffer + audioData
+            let chunkSize = 4096
+
+            while completeData.count >= chunkSize {
+                let chunk = completeData.prefix(chunkSize)
+                audioWebSocket?.sendDataToServer(chunk)
+                print("Sent 4096 bytes of audio.")
+                completeData.removeFirst(chunkSize)
+            }
+
+            partialBuffer = completeData
+        }
+    }
+
+    /// Converts the audio buffer to Float32 Data with RMS normalization and soft clipping.
+    func convertToFloat32BytesLikePython(_ buffer: AVAudioPCMBuffer) -> Data? {
+        guard let int16ChannelData = buffer.int16ChannelData else {
+            print("int16ChannelData is nil.")
+            return nil
+        }
+
+        let frameLength = Int(buffer.frameLength)
+        let channelPointer = int16ChannelData.pointee
+
+        var floatArray = [Float32](repeating: 0, count: frameLength)
+        for i in 0..<frameLength {
+            let int16Value = channelPointer[i]
+            floatArray[i] = Float32(Int16(littleEndian: int16Value)) / 32768.0
+        }
+
+        let rms = sqrt(floatArray.map { $0 * $0 }.reduce(0, +) / Float(frameLength))
+        let targetRMS: Float32 = 0.25
+        let gain = targetRMS / max(rms, 0.00001)
+
+        print("Original RMS: \(rms), applied gain: \(gain)")
+
+        for i in 0..<frameLength {
+            let scaled = floatArray[i] * gain
+            let clipped = tanh(scaled * 3.0)
+            floatArray[i] = clipped
+        }
+
+        let floatData = Data(bytes: floatArray, count: frameLength * MemoryLayout<Float32>.size)
+
+        if let minVal = floatArray.min(), let maxVal = floatArray.max() {
+            print("Float32 value range after normalization: \(minVal)...\(maxVal)")
+        }
+
+        print("Converted to Float32 data: \(floatData.count) bytes")
+        return floatData
+    }
+
+    /// Pauses audio streaming by removing the input tap.
+    func pauseStreaming() {
+        guard !isPaused else { return }
+        inputNode.removeTap(onBus: 0)
+        isPaused = true
+        print("Audio streaming paused.")
+    }
+
+    /// Resumes audio streaming by reinstalling the input tap.
+    func resumeStreaming() {
+        guard isPaused else { return }
+        guard let inputFormat = inputFormat else {
+            print("inputFormat is nil.")
+            return
+        }
+
+        inputNode.installTap(onBus: 0, bufferSize: bufferSize, format: inputFormat) { [weak self] buffer, _ in
+            self?.processAudioBuffer(buffer)
+        }
+        isPaused = false
+        print("Audio streaming resumed.")
+    }
+
+    /// Stops the AVAudioEngine and resets streaming state.
+    func stopStreaming() {
+        guard isStreaming else {
+            print("Already stopped.")
+            return
+        }
+
+        inputNode.removeTap(onBus: 0)
+        engine.stop()
+        isStreaming = false
+        print("AVAudioEngine stopped.")
+    }
+}

Audio-Transcription-iOS/AudioWebSocket.swift

@@ -0,0 +1,256 @@
+//
+//  RecordingViewModel.swift
+//  Lecture2Quiz
+//
+//  Created by ParkMazorika on 4/27/25.
+//
+
+
+import Foundation
+
+/// WebSocket client that connects to a transcription server and handles streaming, JSON messages, and retries.
+class AudioWebSocket: NSObject, URLSessionWebSocketDelegate {
+    private var webSocketTask: URLSessionWebSocketTask?
+    private var urlSession: URLSession!
+    private let host: String
+    private let port: Int
+    private var retryCount = 0
+    private let maxRetries = 3
+    private var uid: String
+    private let modelSize: String
+    private var pingTimer: Timer?
+    private var processedTexts = Set<String>()
+
+    var onServerReady: (() -> Void)?
+    var onTranscriptionReceived: ((String) -> Void)?
+
+    init(host: String, port: Int, modelSize: String = "medium") {
+        self.host = host
+        self.port = port
+        self.uid = UUID().uuidString
+        self.modelSize = modelSize
+        super.init()
+
+        self.urlSession = URLSession(
+            configuration: .default,
+            delegate: self,
+            delegateQueue: .main
+        )
+        connect()
+    }
+
+    /// Establishes a WebSocket connection with the configured server.
+    private func connect() {
+        guard retryCount <= maxRetries else {
+            print("Maximum reconnect attempts exceeded.")
+            return
+        }
+
+        let socketURL = port == 443 || port == 80
+            ? "wss://\(host)"
+            : "wss://\(host):\(port)"
+
+        guard let url = URL(string: socketURL) else {
+            print("Invalid URL: \(socketURL)")
+            return
+        }
+
+        webSocketTask = urlSession.webSocketTask(with: url)
+        webSocketTask?.resume()
+        print("Attempting WebSocket connection: \(socketURL)")
+
+        listen()
+        sendInitialJSON()
+        startPing()
+    }
+
+    /// Sends the initial JSON payload to identify and configure the session.
+    private func sendInitialJSON() {
+        let jsonPayload: [String: Any] = [
+            "uid": uid,
+            "language": "en",
+            "task": "transcribe",
+            "model": modelSize,
+            "use_vad": true,
+            "max_clients": 4,
+            "max_connection_time": 600
+        ]
+
+        do {
+            let jsonData = try JSONSerialization.data(withJSONObject: jsonPayload, options: [])
+            let jsonString = String(data: jsonData, encoding: .utf8) ?? ""
+            print("Sending config JSON: \(jsonString)")
+
+            webSocketTask?.send(.string(jsonString)) { [weak self] error in
+                if let error = error {
+                    print("Failed to send config JSON: \(error.localizedDescription)")
+                    self?.reconnect()
+                } else {
+                    print("Config JSON sent successfully.")
+                }
+            }
+        } catch {
+            print("JSON serialization error: \(error.localizedDescription)")
+        }
+    }
+
+    /// Sends audio data to the server.
+    func sendDataToServer(_ data: Data) {
+        guard isConnected else {
+            print("Not connected - skipping data send.")
+            reconnect()
+            return
+        }
+
+        webSocketTask?.send(.data(data)) { [weak self] error in
+            if let error = error {
+                print("Failed to send audio data: \(error.localizedDescription)")
+                self?.reconnect()
+            } else {
+                print("Sent audio data: \(data.count) bytes")
+            }
+        }
+    }
+
+    /// Returns true if the WebSocket is currently connected.
+    internal var isConnected: Bool {
+        webSocketTask?.state == .running
+    }
+
+    /// Attempts reconnection with exponential backoff.
+    private func reconnect() {
+        retryCount += 1
+        stopPing()
+        let delay = min(5.0, pow(2.0, Double(retryCount)))
+
+        DispatchQueue.global().asyncAfter(deadline: .now() + delay) { [weak self] in
+            print("Reconnecting... (\(self?.retryCount ?? 0)/\(self?.maxRetries ?? 0))")
+            self?.connect()
+        }
+    }
+
+    /// Starts listening for incoming messages from the server.
+    private func listen() {
+        webSocketTask?.receive { [weak self] result in
+            switch result {
+            case .success(let message):
+                self?.handleMessage(message)
+                self?.listen()
+            case .failure(let error):
+                print("Receive error: \(error.localizedDescription)")
+                self?.reconnect()
+            }
+        }
+    }
+
+    /// Handles incoming WebSocket messages (text or binary).
+    private func handleMessage(_ message: URLSessionWebSocketTask.Message) {
+        switch message {
+        case .data(let data):
+            print("Received binary data: \(data.count) bytes")
+
+        case .string(let text):
+            print("Received text message: \(text)")
+
+            guard let data = text.data(using: .utf8) else { return }
+
+            do {
+                if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] {
+                    if let status = json["status"] as? String {
+                        handleStatusMessage(status: status, message: json["message"] as? String)
+                        return
+                    }
+
+                    if let message = json["message"] as? String, message == "SERVER_READY" {
+                        print("Server is ready.")
+                        onServerReady?()
+                        return
+                    }
+
+                    if let segments = json["segments"] as? [[String: Any]] {
+                        let wrapped = ["segments": segments]
+                        let segmentData = try JSONSerialization.data(withJSONObject: wrapped, options: [])
+                        let segmentString = String(data: segmentData, encoding: .utf8)!
+                        onTranscriptionReceived?(segmentString)
+                        print("Transcription segments forwarded.")
+                    }
+                }
+            } catch {
+                print("JSON parsing error: \(error.localizedDescription)")
+            }
+
+        @unknown default:
+            print("Unknown message type received.")
+        }
+    }
+
+    /// Handles status message JSON from the server.
+    private func handleStatusMessage(status: String, message: String?) {
+        switch status {
+        case "WAIT":
+            print("Waiting: \(message ?? "")")
+        case "ERROR":
+            print("Error: \(message ?? "")")
+        case "WARNING":
+            print("Warning: \(message ?? "")")
+        default:
+            print("\(status): \(message ?? "")")
+        }
+    }
+
+    /// Sends the "END_OF_AUDIO" signal to the server.
+    func sendEndOfAudio() {
+        guard isConnected else {
+            print("Not connected - skipping END_OF_AUDIO.")
+            return
+        }
+
+        webSocketTask?.send(.string("END_OF_AUDIO")) { error in
+            if let error = error {
+                print("Failed to send END_OF_AUDIO: \(error.localizedDescription)")
+            } else {
+                print("END_OF_AUDIO sent.")
+            }
+        }
+    }
+
+    /// Gracefully closes the WebSocket connection.
+    func closeConnection() {
+        stopPing()
+        webSocketTask?.cancel(with: .normalClosure, reason: nil)
+        retryCount = maxRetries
+        print("WebSocket closed.")
+    }
+
+    /// Starts periodic ping to keep the WebSocket alive.
+    private func startPing() {
+        stopPing()
+        pingTimer = Timer.scheduledTimer(withTimeInterval: 15.0, repeats: true) { [weak self] _ in
+            self?.webSocketTask?.sendPing { error in
+                if let error = error {
+                    print("Ping failed: \(error.localizedDescription)")
+                } else {
+                    print("Ping sent successfully.")
+                }
+            }
+        }
+        RunLoop.main.add(pingTimer!, forMode: .common)
+    }
+
+    /// Stops the periodic ping timer.
+    private func stopPing() {
+        pingTimer?.invalidate()
+        pingTimer = nil
+    }
+
+    /// Called when the WebSocket is closed by the server.
+    func urlSession(_ session: URLSession,
+                    webSocketTask: URLSessionWebSocketTask,
+                    didCloseWith closeCode: URLSessionWebSocketTask.CloseCode,
+                    reason: Data?) {
+        let reasonString = String(data: reason ?? Data(), encoding: .utf8) ?? "No reason"
+        print("WebSocket closed - code: \(closeCode.rawValue), reason: \(reasonString)")
+        stopPing()
+        reconnect()
+    }
+}

Audio-Transcription-iOS/ContentView.swift

@@ -0,0 +1,99 @@
+//
+//  ContentView.swift
+//  WhisperLive_iOS_Client
+//
+//  Created by ParkMazorika on 6/17/25.
+//
+
+import SwiftUI
+
+/// A standalone view for recording and real-time transcription display.
+struct RecordingView: View {
+    var onDismiss: () -> Void
+    @StateObject private var recordingViewModel = AudioViewModel()
+    @State private var showSubmitView = false
+
+    var body: some View {
+        VStack(spacing: 0) {
+            // Stop button (only visible when recording)
+            HStack {
+                Spacer()
+                if recordingViewModel.isRecording {
+                    Button("Stop Recording") {
+                        recordingViewModel.stopRecording()
+                        recordingViewModel.finalizeTranscription()
+                        showSubmitView = true
+                    }
+                    .font(.headline)
+                    .padding()
+                    .foregroundColor(.gray)
+                }
+            }
+
+            // Transcription display
+            ScrollView {
+                VStack(spacing: 8) {
+                    ForEach(recordingViewModel.transcriptionList.indices, id: \.self) { index in
+                        Text(recordingViewModel.transcriptionList[index])
+                            .padding()
+                            .frame(maxWidth: .infinity, alignment: .leading)
+                            .background(Color.gray.opacity(0.1))
+                            .cornerRadius(8)
+                            .font(.system(size: 14, weight: .semibold))
+                    }
+                }
+                .padding(.horizontal)
+            }
+
+            Divider().padding(.top, 8)
+
+            // Timer and Record/Pause/Resume button
+            VStack(spacing: 16) {
+                Text(recordingViewModel.timeLabel)
+                    .font(.system(size: 40))
+
+                Button(action: {
+                    if recordingViewModel.isRecording {
+                        recordingViewModel.isPaused
+                            ? recordingViewModel.resumeRecording()
+                            : recordingViewModel.pauseRecording()
+                    } else {
+                        recordingViewModel.startRecording()
+                    }
+                }) {
+                    Image(systemName: recordingViewModel.isRecording
+                          ? (recordingViewModel.isPaused ? "play.circle.fill" : "pause.circle.fill")
+                          : "mic.circle.fill")
+                        .font(.system(size: 50))
+                        .foregroundStyle(.black)
+                }
+            }
+            .padding(.bottom, 40)
+        }
+        .padding(.top)
+        .background(Color(.systemBackground))
+        .overlay(
+            Group {
+                if recordingViewModel.isLoading {
+                    ZStack {
+                        Color.black.opacity(0.4).ignoresSafeArea()
+                        ProgressView("Processing...")
+                            .padding()
+                            .background(Color.white)
+                            .cornerRadius(10)
+                    }
+                }
+            }
+        )
+        .sheet(isPresented: $showSubmitView) {
+            //anotherView
+        }
+    }
+}
+
+#Preview("Recording View") {
+    RecordingView {
+        // Dummy dismiss handler
+        print("RecordingView dismissed")
+    }
+}

Audio-Transcription-iOS/README.md

@@ -0,0 +1,109 @@
+# Audio-Transcription-iOS
+
+This is an iOS client for [WhisperLive](https://github.com/collabora/WhisperLive), a real-time speech-to-text server based on OpenAI Whisper.  
+The app streams microphone audio to a WhisperLive server via WebSocket and displays live transcription results in real time.
+
+> ⚠️ This client is designed to work specifically with the [WhisperLive Python WebSocket server](https://github.com/collabora/WhisperLive?tab=readme-ov-file#running-the-server).  
+> Make sure the server is running and reachable from your iOS device.
+
+## Features
+
+- Real-time microphone capture with AVAudioEngine
+- Streaming to WhisperLive backend using WebSocket
+- Displays transcription as segments arrive
+- Start / Pause / Resume / Stop recording with SwiftUI interface
+- Final transcription view on stop
+
+## Requirements
+
+- iOS 15.0+
+- Swift 5.8+
+- AVFoundation (for microphone)
+- Working WhisperLive WebSocket server
+
+## Getting Started
+
+1. Clone the repository (your fork):
+
+    ```bash
+    git clone https://github.com/yourusername/whisperlive.git
+    cd whisperlive/Audio-Transcription-iOS
+    ```
+
+2. Open the `.xcodeproj` or `.xcodeworkspace` in Xcode
+
+3. Add the following to your `Info.plist`:
+
+    ```xml
+    <key>NSMicrophoneUsageDescription</key>
+    <string>This app requires microphone access for transcription.</string>
+    ```
+
+4. Run the app on a physical device (recommended)
+
+## Running on a Physical Device (with Free Apple ID)
+
+You can run this app on a real iPhone without a paid Apple Developer account. Follow these steps:
+
+### 1. Register a Free Apple ID in Xcode
+
+1. Open Xcode ▸ Settings… (or Preferences) ▸ **Accounts**
+2. Click the **+** button ▸ Select **Apple ID**
+3. Sign in with your Apple ID (a free one is fine)
+4. A "Personal Team" will be created automatically
+
+> ✅ You can deploy up to 3 apps on a physical device using a free Apple ID with a 7-day provisioning profile.
+
+---
+
+### 2. Set Up Signing in Your Project
+
+1. In Xcode, select your **project** in the Project Navigator
+2. Go to **TARGETS ▸ YourAppName ▸ Signing & Capabilities**
+3. Set **Team** to your Personal Team
+4. Set a unique **Bundle Identifier** (e.g., `com.yourname.whisperlive`)
+5. Make sure **Automatically manage signing** is checked
+6. If a red warning appears, click **"Resolve Issues"**
+
+---
+
+### 3. Connect and Trust Your iPhone
+
+1. Connect your iPhone via USB
+2. When prompted, tap **“Trust This Computer”** on your iPhone
+3. Make sure your iPhone appears in Xcode's device list
+
+---
+
+### 4. Enable Developer Mode on iPhone
+
+1. Press the **Build (▶︎)** button in Xcode
+2. Your iPhone will ask to enable **Developer Mode**
+3. On iPhone, go to:  
+   **Settings ▸ Privacy & Security ▸ Developer Mode**
+4. Enable it and restart the device if required
+
+---
+
+Now you can run and debug the app on your real device!
+
+
+
+## Folder Structure
+```
+Audio-Transcription-iOS/
+├── AudioViewModel.swift
+├── AudioStreamer.swift
+├── AudioWebSocket.swift
+├── RecordingView.swift
+├── WhisperLive_iOS_ClientApp.swift
+├── Info.plist
+├── README.md
+```
+
+## License
+
+MIT  
+This iOS client is provided as an open-source example to complement WhisperLive's real-time transcription ecosystem.
+
+

Audio-Transcription-iOS/RecordingViewModel.swift

@@ -0,0 +1,174 @@
+//
+//  RecordingViewModel.swift
+//  Lecture2Quiz
+//
+//  Created by ParkMazorika on 4/27/25.
+//
+
+import AVFoundation
+import Combine
+
+/// Represents a segment of transcribed audio with start/end timestamps and completion flag.
+struct TranscriptionSegment: Identifiable, Equatable {
+    var id = UUID()
+    var start: Double
+    var end: Double
+    var text: String
+    var completed: Bool
+}
+
+/// ViewModel responsible for managing audio recording and transcription logic.
+class AudioViewModel: ObservableObject {
+    @Published var isRecording = false            // Indicates if recording is active
+    @Published var isPaused = false               // Indicates if recording is currently paused
+    @Published var timeLabel = "00:00"            // Timer label formatted as mm:ss
+    @Published var transcriptionList: [String] = []  // Live transcription output
+    @Published var isLoading = false              // True while waiting for server response
+    @Published var finalScript: String = ""       // Final script from completed segments
+
+    private var timer: Timer?
+    private var elapsedTime: Int = 0
+
+    private var audioStreamer: AudioStreamer?     // Handles audio capture and streaming
+    private var audioWebSocket: AudioWebSocket?   // Manages WebSocket communication
+
+    private var segments: [TranscriptionSegment] = []  // Stores all transcription segments
+
+    init() {}
+
+    /// Starts audio recording and initializes WebSocket + AVAudioEngine.
+    func startRecording() {
+        let audioAPIUrl = "your server url"
+        audioWebSocket = AudioWebSocket(host: audioAPIUrl, port: 443)
+        audioStreamer = AudioStreamer(webSocket: audioWebSocket!)
+
+        isLoading = true
+
+        // Handle server transcription message
+        audioWebSocket?.onTranscriptionReceived = { [weak self] text in
+            self?.handleRawTranscriptionJSON(text)
+        }
+
+        // When server sends SERVER_READY
+        audioWebSocket?.onServerReady = { [weak self] in
+            guard let self = self else { return }
+            DispatchQueue.main.async {
+                self.isLoading = false
+                self.isRecording = true
+                self.isPaused = false
+                self.timeLabel = "00:00"
+                self.elapsedTime = 0
+                self.startTimer()
+                self.audioStreamer?.startStreaming()
+            }
+        }
+    }
+
+    /// Pauses the recording and stops the timer.
+    func pauseRecording() {
+        isPaused = true
+        audioStreamer?.pauseStreaming()
+        timer?.invalidate()
+    }
+
+    /// Resumes recording and restarts the timer.
+    func resumeRecording() {
+        isPaused = false
+        audioStreamer?.resumeStreaming()
+        startTimer()
+    }
+
+    /// Stops recording and finalizes connection to server.
+    func stopRecording() {
+        isRecording = false
+        isPaused = false
+        timer?.invalidate()
+
+        audioStreamer?.stopStreaming()
+        audioWebSocket?.sendEndOfAudio()
+        audioWebSocket?.onTranscriptionReceived = nil
+        audioWebSocket?.closeConnection()
+    }
+
+    /// Starts the recording timer (1-second interval).
+    private func startTimer() {
+        timer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { _ in
+            self.elapsedTime += 1
+            let minutes = self.elapsedTime / 60
+            let seconds = self.elapsedTime % 60
+            self.timeLabel = String(format: "%02d:%02d", minutes, seconds)
+        }
+    }
+
+    /// Finalizes the transcription by joining all completed segments into one string.
+    func finalizeTranscription() {
+        isLoading = false
+        let completedText = segments
+            .filter { $0.completed }
+            .map { $0.text.trimmingCharacters(in: .whitespaces) }
+            .joined(separator: " ")
+        finalScript = completedText
+        print("Final transcript:\n\(finalScript)")
+    }
+
+    /// Handles incoming JSON from the server and updates UI state.
+    /// Supports both full JSON and raw string cases.
+    func handleRawTranscriptionJSON(_ jsonString: String) {
+        let trimmed = jsonString.trimmingCharacters(in: .whitespacesAndNewlines)
+        guard let data = trimmed.data(using: .utf8) else { return }
+
+        if trimmed.hasPrefix("{") {
+            // Parse JSON containing segment list
+            do {
+                if let dict = try JSONSerialization.jsonObject(with: data) as? [String: Any],
+                   let segmentDicts = dict["segments"] as? [[String: Any]] {
+
+                    for item in segmentDicts {
+                        guard let startStr = item["start"] as? String,
+                              let endStr = item["end"] as? String,
+                              let text = item["text"] as? String,
+                              let completed = item["completed"] as? Bool,
+                              let start = Double(startStr),
+                              let end = Double(endStr) else { continue }
+
+                        let newSegment = TranscriptionSegment(start: start, end: end, text: text, completed: completed)
+
+                        // Overwrite if already exists, else append
+                        if let index = self.segments.firstIndex(where: { $0.start == start }) {
+                            self.segments[index] = newSegment
+                        } else {
+                            self.segments.append(newSegment)
+                        }
+                    }
+
+                    // Update the UI
+                    DispatchQueue.main.async {
+                        let completedTexts = self.segments
+                            .filter { $0.completed }
+                            .sorted(by: { $0.start < $1.start })
+                            .map { $0.text.trimmingCharacters(in: .whitespaces) }
+
+                        let pendingText = self.segments
+                            .filter { !$0.completed }
+                            .sorted(by: { $0.start < $1.start })
+                            .map { $0.text.trimmingCharacters(in: .whitespaces) }
+                            .last ?? ""
+
+                        self.transcriptionList = completedTexts + (pendingText.isEmpty ? [] : [pendingText])
+                        self.finalScript = self.transcriptionList.joined(separator: " ")
+                    }
+                }
+            } catch {
+                print("JSON parsing error: \(error)")
+            }
+        } else {
+            // Handle raw text line
+            DispatchQueue.main.async {
+                if self.transcriptionList.last != trimmed {
+                    self.transcriptionList.append(trimmed)
+                    self.finalScript = self.transcriptionList.joined(separator: " ")
+                }
+            }
+        }
+    }
+}

Audio-Transcription-iOS/WhisperLive-iOS-Client-Info.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>NSMicrophoneUsageDescription</key>
+	<string>This app requires microphone access for voice transcription.</string>
+</dict>
+</plist>

Audio-Transcription-iOS/WhisperLive_iOS_ClientApp.swift

@@ -0,0 +1,20 @@
+//
+//  WhisperLive_iOS_ClientApp.swift
+//  WhisperLive_iOS_Client
+//
+//  Created by 바견규 on 6/17/25.
+//
+
+import SwiftUI
+
+@main
+struct WhisperLive_iOS_ClientApp: App {
+    var body: some Scene {
+        WindowGroup {
+            RecordingView {
+                // Handle dismiss action here, or leave it empty for now
+                print("RecordingView dismissed")
+            }
+        }
+    }
+}

README.md

@@ -3,6 +3,8 @@
 <h2 align="center">
   <a href="https://www.youtube.com/watch?v=0PHWCApIcCI"><img
 src="https://img.youtube.com/vi/0PHWCApIcCI/0.jpg" style="background-color:rgba(0,0,0,0);" height=300 alt="WhisperLive"></a>
+  <a href="https://www.youtube.com/watch?v=0f5oiG4oPWQ"><img
+  src="https://img.youtube.com/vi/0f5oiG4oPWQ/0.jpg" style="background-color:rgba(0,0,0,0);" height=300 alt="WhisperLive"></a>
   <br><br>A nearly-live implementation of OpenAI's Whisper.
 <br><br>
 </h2>
@@ -11,8 +13,19 @@ This project is a real-time transcription application that uses the OpenAI Whisp
 to convert speech input into text output. It can be used to transcribe both live audio
 input from microphone and pre-recorded audio files.
 
+- [Installation](#installation)
+- [Getting Started](#getting-started)
+- [Running the Server](#running-the-server)
+- [Running the Client](#running-the-client)
+- [Browser Extensions](#browser-extensions)
+- [Whisper Live Server in Docker](#whisper-live-server-in-docker)
+- [Future Work](#future-work)
+- [Blog Posts](#blog-posts)
+- [Contact](#contact)
+- [Citations](#citations)
+
 ## Installation
-- Install PyAudio and ffmpeg
+- Install PortAudio
 ```bash
  bash scripts/setup.sh
 ```
@@ -22,22 +35,53 @@ input from microphone and pre-recorded audio files.
  pip install whisper-live
 ```
 
+
+- Install 3.12 venv on Fedora
+
+```bash
+sudo dnf install -y python3.12 python3.12-pip
+python3.12 -m venv whisper_env
+source whisper_env/bin/activate
+```
+
+
+### OpenAI REST interface
+
+#### Server
+
+```bash
+python3 run_server.py --port 9090 --backend faster_whisper --max_clients 4 --max_connection_time 600 --enable_rest --cors-origins="http://localhost:8080,http://127.0.0.1:8080"
+```
+
+#### Client
+
+```bash
+python3 client_openai.py $AUDIO_FILE
+```
+
+
+
 ### Setting up NVIDIA/TensorRT-LLM for TensorRT backend
 - Please follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md) for setup of [NVIDIA/TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM) and for building Whisper-TensorRT engine.
 
 ## Getting Started
-The server supports two backends `faster_whisper` and `tensorrt`. If running `tensorrt` backend follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md)
+The server supports 3 backends `faster_whisper`, `tensorrt` and `openvino`. If running `tensorrt` backend follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md)
 
 ### Running the Server
 - [Faster Whisper](https://github.com/SYSTRAN/faster-whisper) backend
 ```bash
 python3 run_server.py --port 9090 \
-                      --backend faster_whisper
+                      --backend faster_whisper \
+                      --max_clients 4 \
+                      --max_connection_time 600
   
-# running with custom model
+# running with custom model and cache_dir to save auto-converted ctranslate2 models
 python3 run_server.py --port 9090 \
-                      --backend faster_whisper
-                      -fw "/path/to/custom/faster/whisper/model"
+                      --backend faster_whisper \
+                      --max_clients 4 \
+                      --max_connection_time 600 \
+                      -fw "/path/to/custom/faster/whisper/model" \
+                      -c ~/.cache/whisper-live/
 ```
 
 - TensorRT backend. Currently, we recommend to only use the docker setup for TensorRT. Follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md) which works as expected. Make sure to build your TensorRT Engines before running the server with TensorRT backend.
@@ -45,18 +89,65 @@ python3 run_server.py --port 9090 \
 # Run English only model
 python3 run_server.py -p 9090 \
                       -b tensorrt \
-                      -trt /home/TensorRT-LLM/examples/whisper/whisper_small_en
+                      -trt /home/TensorRT-LLM/examples/whisper/whisper_small_en \
+                      --max_clients 4 \
+                      --max_connection_time 600
 
 # Run Multilingual model
 python3 run_server.py -p 9090 \
                       -b tensorrt \
                       -trt /home/TensorRT-LLM/examples/whisper/whisper_small \
-                      -m
+                      -m \
+                      --max_clients 4 \
+                      --max_connection_time 600
+```
+- Use `--max_clients` option to restrict the number of clients the server should allow. Defaults to 4.
+- Use `--max_connection_time` options to limit connection time for a client in seconds. Defaults to 600.
+- WhisperLive now supports the [OpenVINO](https://github.com/openvinotoolkit/openvino) backend for efficient inference on Intel CPUs, iGPU and dGPUs. Currently, we tested the models uploaded to [huggingface by OpenVINO](https://huggingface.co/OpenVINO?search_models=whisper).
+  - > **Docker Recommended:** Running WhisperLive with OpenVINO inside Docker automatically enables GPU support (iGPU/dGPU) without requiring additional host setup.
+  - > **Native (non-Docker) Use:** If you prefer running outside Docker, ensure the Intel drivers and OpenVINO runtime are installed and properly configured on your system. Refer to the documentation for [installing OpenVINO](https://docs.openvino.ai/2025/get-started/install-openvino.html?PACKAGE=OPENVINO_BASE&VERSION=v_2025_0_0&OP_SYSTEM=LINUX&DISTRIBUTION=PIP#).
+
+```
+python3 run_server.py -p 9090 -b openvino
 ```
 
 
+#### Controlling OpenMP Threads
+To control the number of threads used by OpenMP, you can set the `OMP_NUM_THREADS` environment variable. This is useful for managing CPU resources and ensuring consistent performance. If not specified, `OMP_NUM_THREADS` is set to `1` by default. You can change this by using the `--omp_num_threads` argument:
+```bash
+python3 run_server.py --port 9090 \
+                      --backend faster_whisper \
+                      --omp_num_threads 4
+```
+
+#### Single model mode
+By default, when running the server without specifying a model, the server will instantiate a new whisper model for every client connection. This has the advantage, that the server can use different model sizes, based on the client's requested model size. On the other hand, it also means you have to wait for the model to be loaded upon client connection and you will have increased (V)RAM usage.
+
+When serving a custom TensorRT model using the `-trt` or a custom faster_whisper model using the `-fw` option, the server will instead only instantiate the custom model once and then reuse it for all client connections.
+
+If you don't want this, set `--no_single_model`.
+
+
 ### Running the Client
-- Initializing the client:
+
+Use the below command to run the client:
+```bash
+python3 run_client.py --files <audio-file-name>
+```
+This will connect to the localhost server running on port 9090 by default. Use flags `--server` and `--port` to use different configurations. The above command will transcribe audio file provided with `--files` flag.
+
+
+Here are the details of client instance implemented in `run_client.py` script:
+  - `lang`: Language of the input audio, applicable only if using a multilingual model.
+  - `translate`: If set to `True` then translate from any language to `en`.
+  - `model`: Whisper model size.
+  - `use_vad`: Whether to use `Voice Activity Detection` on the server.
+  - `save_output_recording`: Set to True to save the microphone input as a `.wav` file during live transcription. This option is helpful for recording sessions for later playback or analysis. Defaults to `False`. 
+  - `output_recording_filename`: Specifies the `.wav` file path where the microphone input will be saved if `save_output_recording` is set to `True`.
+  - `mute_audio_playback`: Whether to mute audio playback when transcribing an audio file. Defaults to False.
+  - `enable_translation`: Start translation thread on the server (from any to any).
+  - `target_language`: Server translation thread's target translation language.
+
 ```python
 from whisper_live.client import TranscriptionClient
 client = TranscriptionClient(
@@ -64,13 +155,18 @@ client = TranscriptionClient(
   9090,
   lang="en",
   translate=False,
-  model="small",
+  model="small",                                      # also support hf_model => `Systran/faster-whisper-small`
   use_vad=False,
+  save_output_recording=True,                         # Only used for microphone input, False by Default
+  output_recording_filename="./output_recording.wav", # Only used for microphone input
+  mute_audio_playback=False,                          # Only used for file input, False by Default
+  enable_translation=True,
+  target_language="hi",
 )
 ```
 It connects to the server running on localhost at port 9090. Using a multilingual model, language for the transcription will be automatically detected. You can also use the language option to specify the target language for the transcription, in this case, English ("en"). The translate option should be set to `True` if we want to translate from the source language to English and `False` if we want to transcribe in the source language.
 
-- Trancribe an audio file:
+- Transcribe an audio file:
 ```python
 client("tests/jfk.wav")
 ```
@@ -80,14 +176,25 @@ client("tests/jfk.wav")
 client()
 ```
 
+- To transcribe from a RTSP stream:
+```python
+client(rtsp_url="rtsp://admin:admin@192.168.0.1/rtsp")
+```
+
 - To transcribe from a HLS stream:
 ```python
-client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/bbc_1xtra.isml/bbc_1xtra-audio%3d96000.norewind.m3u8") 
+client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/bbc_1xtra.isml/bbc_1xtra-audio%3d96000.norewind.m3u8")
 ```
 
 ## Browser Extensions
 - Run the server with your desired backend as shown [here](https://github.com/collabora/WhisperLive?tab=readme-ov-file#running-the-server).
-- Transcribe audio directly from your browser using our Chrome or Firefox extensions. Refer to [Audio-Transcription-Chrome](https://github.com/collabora/whisper-live/tree/main/Audio-Transcription-Chrome#readme) and [Audio-Transcription-Firefox](https://github.com/collabora/whisper-live/tree/main/Audio-Transcription-Firefox#readme) for setup instructions.
+- Transcribe audio directly from your browser using our Chrome or Firefox extensions. Refer to [Audio-Transcription-Chrome](https://github.com/collabora/whisper-live/tree/main/Audio-Transcription-Chrome#readme) and https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md
+
+## iOS Client
+
+Use WhisperLive on iOS with our native iOS client.  
+Refer to [`ios-client`](https://github.com/collabora/WhisperLive/tree/main/Audio-Transcription-iOS) and [`ios-client/README.md`](https://github.com/collabora/WhisperLive/blob/main/Audio-Transcription-iOS/README.md) for setup and usage instructions.
+
 
 ## Whisper Live Server in Docker
 - GPU
@@ -96,22 +203,48 @@ client(hls_url="http://as-hls-ww-live.akamaized.net/pool_904/live/ww/bbc_1xtra/b
   docker run -it --gpus all -p 9090:9090 ghcr.io/collabora/whisperlive-gpu:latest
   ```
 
-  - TensorRT. Follow [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md) in order to setup docker and use TensorRT backend. We provide a pre-built docker image which has TensorRT-LLM built and ready to use.
+  - TensorRT. Refer to [TensorRT_whisper readme](https://github.com/collabora/WhisperLive/blob/main/TensorRT_whisper.md) for setup and more tensorrt backend configurations.
+  ```bash
+  docker build . -f docker/Dockerfile.tensorrt -t whisperlive-tensorrt
+  docker run -p 9090:9090 --runtime=nvidia --entrypoint /bin/bash -it whisperlive-tensorrt
+
+  # Build small.en engine
+  bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small.en        # float16
+  bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small.en int8   # int8 weight only quantization
+  bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small.en int4   # int4 weight only quantization
+
+  # Run server with small.en
+  python3 run_server.py --port 9090 \
+                        --backend tensorrt \
+                        --trt_model_path "/app/TensorRT-LLM-examples/whisper/whisper_small_en_float16"
+                        --trt_model_path "/app/TensorRT-LLM-examples/whisper/whisper_small_en_int8"
+                        --trt_model_path "/app/TensorRT-LLM-examples/whisper/whisper_small_en_int4"
+  ```
+
+  - OpenVINO
+  ```
+  docker run -it --device=/dev/dri -p 9090:9090 ghcr.io/collabora/whisperlive-openvino
+  ```
 
 - CPU
-```bash
-docker run -it -p 9090:9090 ghcr.io/collabora/whisperlive-cpu:latest
-```
-**Note**: By default we use "small" model size. To build docker image for a different model size, change the size in server.py and then build the docker image.
+  - Faster-whisper
+  ```bash
+  docker run -it -p 9090:9090 ghcr.io/collabora/whisperlive-cpu:latest
+  ```
 
 ## Future Work
-- [ ] Add translation to other languages on top of transcription.
-- [x] TensorRT backend for Whisper.
+- [x] Add translation to other languages on top of transcription.
+
+## Blog Posts
+- [Transforming speech technology with WhisperLive](https://www.collabora.com/news-and-blog/blog/2024/05/28/transforming-speech-technology-with-whisperlive/)
+- [WhisperFusion: Ultra-low latency conversations with an AI chatbot](https://www.collabora.com/news-and-blog/news-and-events/whisperfusion-ultra-low-latency-conversations-with-an-ai-chatbot.html) powered by WhisperLive
+- [Breaking language barriers 2.0: Moving closer towards fully reliable, production-ready Hindi ASR](https://www.collabora.com/news-and-blog/news-and-events/breaking-language-barriers-20-moving-closer-production-ready-hindi-asr.html) which is used in WhisperLive for hindi.
 
 ## Contact
 
 We are available to help you with both Open Source and proprietary AI projects. You can reach us via the Collabora website or [vineet.suryan@collabora.com](mailto:vineet.suryan@collabora.com) and [marcus.edel@collabora.com](mailto:marcus.edel@collabora.com).
 
+
 ## Citations
 ```bibtex
 @article{Whisper

TensorRT_whisper.md

@@ -1,67 +1,47 @@
-# Whisper-TensorRT
+# WhisperLive-TensorRT
 We have only tested the TensorRT backend in docker so, we recommend docker for a smooth TensorRT backend setup.
-**Note**: We use [our fork to setup TensorRT](https://github.com/makaveli10/TensorRT-LLM)
+**Note**: We use `tensorrt_llm==0.18.2`
 
 ## Installation
 - Install [docker](https://docs.docker.com/engine/install/)
 - Install [nvidia-container-toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
 
-- Clone this repo.
+- Run WhisperLive TensorRT in docker
 ```bash
-git clone https://github.com/collabora/WhisperLive.git
-cd WhisperLive
+docker build . -f docker/Dockerfile.tensorrt -t whisperlive-tensorrt
+docker run -p 9090:9090 --runtime=nvidia --gpus all --entrypoint /bin/bash -it whisperlive-tensorrt
 ```
 
-- Pull the TensorRT-LLM docker image which we prebuilt for WhisperLive TensorRT backend.
-```bash
-docker pull ghcr.io/collabora/whisperbot-base:latest
-```
-
-- Next, we run the docker image and mount WhisperLive repo to the containers `/home` directory.
-```bash
-docker run -it --gpus all --shm-size=8g \
-       --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \
-       -p 9090:9090 -v /path/to/WhisperLive:/home/WhisperLive \
-       ghcr.io/collabora/whisperbot-base:latest
-```
-
-- Make sure to test the installation. 
-```bash
-# export ENV=${ENV:-/etc/shinit_v2} 
-# source $ENV
-python -c "import torch; import tensorrt; import tensorrt_llm"
-```
-**NOTE**: Uncomment and update library paths if imports fail.
-
 ## Whisper TensorRT Engine
-- We build `small.en` and `small` multilingual TensorRT engine. The script logs the path of the directory with Whisper TensorRT engine. We need the model_path to run the server.
+- We build `small.en` and `small` multilingual TensorRT engine as examples below. The script logs the path of the directory with Whisper TensorRT engine. We need that model_path to run the server.
 ```bash
 # convert small.en
-bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small.en
+bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small.en        # float16
+bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small.en int8   # int8 weight only quantization
+bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small.en int4   # int4 weight only quantization
 
 # convert small multilingual model
-bash scripts/build_whisper_tensorrt.sh /root/TensorRT-LLM-examples small
+bash build_whisper_tensorrt.sh /app/TensorRT-LLM-examples small
 ```
 
 ## Run WhisperLive Server with TensorRT Backend
 ```bash
-cd /home/WhisperLive
-
-# Install requirements
-apt update && bash scripts/setup.sh
-pip install -r requirements/server.txt
-
-# Required to create mel spectogram
-wget --directory-prefix=assets assets/mel_filters.npz https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz
-
 # Run English only model
 python3 run_server.py --port 9090 \
                       --backend tensorrt \
-                      --trt_model_path "path/to/whisper_trt/from/build/step"
+                      --trt_model_path "/app/TensorRT-LLM-examples/whisper/whisper_small_en_float16"
 
 # Run Multilingual model
 python3 run_server.py --port 9090 \
                       --backend tensorrt \
-                      --trt_model_path "path/to/whisper_trt/from/build/step" \
+                      --trt_model_path "/app/TensorRT-LLM-examples/whisper/whisper_small_float16" \
                       --trt_multilingual
 ```
+
+By default trt_backend uses cpp_session, to use python session pass `--trt_py_session` to run_server.py
+```bash
+python3 run_server.py --port 9090 \
+                      --backend tensorrt \
+                      --trt_model_path "/app/TensorRT-LLM-examples/whisper/whisper_small_float16" \
+                      --trt_py_session
+```

client_openai.py

@@ -0,0 +1,38 @@
+import sys
+import requests
+
+if len(sys.argv) < 2:
+    print("Usage: python transcribe_file.py <path_to_audio_file>")
+    sys.exit(1)
+
+audio_file = sys.argv[1]
+
+# Configuration
+host = "localhost"
+port = 8000  # Default REST port; change if you used --rest_port
+url = f"http://{host}:{port}/v1/audio/transcriptions"
+model = "small"  # Or "whisper-1" (mapped to small internally)
+language = "en"  # Or "hi" for Hindi
+response_format = "json"  # Options: "json", "text", "verbose_json", "srt", "vtt"
+
+# Prepare the request
+files = {"file": open(audio_file, "rb")}
+data = {
+    "model": model,
+    "language": language,
+    "response_format": response_format,
+    # Optional: Add "prompt" for style guidance, "temperature" (0-1), etc.
+}
+
+# Send the request
+response = requests.post(url, files=files, data=data)
+
+if response.status_code == 200:
+    if response_format == "json" or response_format == "verbose_json":
+        result = response.json()
+        print("Transcript:", result.get("text", "No text found"))
+        # If you need translation, post-process here (e.g., using another API like Google Translate)
+    else:
+        print("Transcript:", response.text)
+else:
+    print("Error:", response.status_code, response.json().get("error", "Unknown error"))

docker/Dockerfile.cpu

@@ -1,22 +1,23 @@
-FROM python:3.8-slim-buster
+FROM python:3.10-bookworm
 
 ARG DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl \
-    ca-certificates \
-    sudo \
-    git \
-    bzip2 \
-    libx11-6 \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
+# install lib required for pyaudio
+RUN apt update && apt install -y portaudio19-dev && apt-get clean && rm -rf /var/lib/apt/lists/*
 
+# update pip to support for whl.metadata -> less downloading
+RUN pip install --no-cache-dir -U "pip>=24"
+
+# create a working directory
+RUN mkdir /app
 WORKDIR /app
 
-COPY scripts/setup.sh requirements/server.txt /app/
+# install pytorch, but without the nvidia-libs that are only necessary for gpu
+RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu
 
-RUN apt update && bash setup.sh && pip install -r server.txt
+# install the requirements for running the whisper-live server
+COPY requirements/server.txt /app/
+RUN pip install --no-cache-dir -r server.txt && rm server.txt
 
 COPY whisper_live /app/whisper_live
 COPY run_server.py /app

docker/Dockerfile.gpu

@@ -1,33 +1,26 @@
-FROM nvidia/cuda:11.8.0-cudnn8-runtime-ubuntu22.04
+FROM python:3.10-bookworm
+
 ARG DEBIAN_FRONTEND=noninteractive
 
-# Remove any third-party apt sources to avoid issues with expiring keys.
-RUN rm -f /etc/apt/sources.list.d/*.list
+# install lib required for pyaudio
+RUN apt update && apt install -y portaudio19-dev && apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# Install some basic utilities.
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    curl \
-    ca-certificates \
-    sudo \
-    git \
-    bzip2 \
-    libx11-6 \
-    python3-dev \
-    python3-pip \
-    && python3 -m pip install --upgrade pip \
-    && rm -rf /var/lib/apt/lists/*
+# update pip to support for whl.metadata -> less downloading
+RUN pip install --no-cache-dir -U "pip>=24"
 
-# Create a working directory.
+# create a working directory
 RUN mkdir /app
 WORKDIR /app
 
-COPY scripts/setup.sh requirements/server.txt /app
+# install the requirements for running the whisper-live server
+COPY requirements/server.txt /app/
+RUN pip install --no-cache-dir -r server.txt && rm server.txt
 
-RUN apt update && bash setup.sh && rm setup.sh
-RUN pip install -r server.txt && rm server.txt
+# make the paths of the nvidia libs installed as wheels visible. equivalent to:
+# export LD_LIBRARY_PATH=`python3 -c 'import os; import nvidia.cublas.lib; import nvidia.cudnn.lib; print(os.path.dirname(nvidia.cublas.lib.__file__) + ":" + os.path.dirname(nvidia.cudnn.lib.__file__))'`
+ENV LD_LIBRARY_PATH="/usr/local/lib/python3.10/site-packages/nvidia/cublas/lib:/usr/local/lib/python3.10/site-packages/nvidia/cudnn/lib"
 
 COPY whisper_live /app/whisper_live
-
 COPY run_server.py /app
 
 CMD ["python", "run_server.py"]

docker/Dockerfile.openvino

@@ -0,0 +1,19 @@
+FROM openvino/ubuntu22_runtime:latest
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+USER root
+
+RUN apt update && apt install -y portaudio19-dev python-is-python3 && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir -U "pip>=24"
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY requirements/server.txt /app/
+RUN pip install --no-cache-dir -r server.txt && rm server.txt
+
+COPY whisper_live /app/whisper_live
+COPY run_server.py /app
+CMD ["python", "run_server.py", "--backend", "openvino"]

docker/Dockerfile.tensorrt

@@ -0,0 +1,30 @@
+FROM nvidia/cuda:12.8.1-base-ubuntu22.04 AS base
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    python3.10 python3-pip openmpi-bin libopenmpi-dev git git-lfs wget \
+    && apt install python-is-python3 \
+    && pip install --upgrade pip setuptools \
+    && rm -rf /var/lib/apt/lists/*
+
+FROM base AS devel
+RUN pip install --no-cache-dir -U tensorrt_llm==0.18.2 --extra-index-url https://pypi.nvidia.com
+WORKDIR /app
+RUN git clone -b v0.18.2 https://github.com/NVIDIA/TensorRT-LLM.git \
+    && mv TensorRT-LLM/examples ./TensorRT-LLM-examples \
+    && rm -rf TensorRT-LLM
+
+FROM devel AS release
+WORKDIR /app
+COPY assets/ ./assets
+RUN wget -nc -P assets/ https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz
+
+COPY scripts/setup.sh ./
+RUN apt update && bash setup.sh && rm setup.sh
+
+COPY requirements/server.txt .
+RUN pip install --no-cache-dir -r server.txt && rm server.txt
+COPY whisper_live ./whisper_live
+COPY scripts/build_whisper_tensorrt.sh .
+COPY run_server.py .

requirements/client.txt

@@ -1,4 +1,4 @@
 PyAudio
-ffmpeg-python
+av
 scipy
 websocket-client

requirements/server.txt

@@ -1,12 +1,27 @@
-faster-whisper==0.10.0
-torch
+faster-whisper==1.2.0
 websockets
-onnxruntime==1.16.0
+onnxruntime==1.17.0
 numba
-openai-whisper
 kaldialign
 soundfile
-ffmpeg-python
 scipy
+av
 jiwer
-evaluate
+evaluate
+numpy<2
+openai-whisper==20250625
+tokenizers==0.20.3
+transformers[torch]
+sentencepiece
+
+# openvino
+librosa
+openvino
+openvino-genai
+openvino-tokenizers
+optimum 
+optimum-intel
+
+fastapi
+uvicorn
+python-multipart

run_client.py

@@ -0,0 +1,98 @@
+from pathlib import Path
+import sys
+from whisper_live.client import TranscriptionClient
+import argparse
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--port', '-p',
+                          type=int,
+                          default=9090,
+                          help="Websocket port to run the server on.")
+    parser.add_argument('--server', '-s',
+                          type=str,
+                          default='localhost',
+                          help='hostname or ip address of server')
+    parser.add_argument('--files', '-f',
+                          type=str,
+                          nargs='+',
+                          help='Files to transcribe, separated by spaces. '
+                              'If not provided, will use microphone input.')
+    parser.add_argument('--output_file', '-o',
+                          type=str,
+                          default='./output_recording.wav',
+                          help='output recording filename, only used for microphone input.')
+    parser.add_argument('--model', '-m',
+                          type=str,
+                          default='small',
+                          help='Model to use for transcription, e.g., "tiny, small.en, large-v3".')
+    parser.add_argument('--lang', '-l',
+                          type=str,
+                          default='en',
+                          help='Language code for transcription, e.g., "en" for English.')
+    parser.add_argument('--translate', '-t',
+                          action='store_true',
+                          help='Enable translation of the transcription output.')
+    parser.add_argument('--mute_audio_playback', '-a',
+                          action='store_true',
+                          help='Mute audio playback during transcription.') 
+    parser.add_argument('--save_output_recording', '-r',
+                          action='store_true',
+                          help='Save the output recording, only used for microphone input.')
+    parser.add_argument('--enable_translation',
+                          action='store_true',
+                          help='Enable translation of the transcription output.')
+    parser.add_argument('--target_language', '-tl',
+                          type=str,
+                          default='fr',
+                          help='Target language for translation, e.g., "fr" for French.')
+    parser.add_argument('--enable_timestamps',
+                          action='store_true',
+                          help='Show transcription with timestamps')
+    parser.add_argument('--n_display_segments',
+                          type=int,
+                          default=4,
+                          help='Number of transcript segments to display in terminal (default: 4).')
+
+    args = parser.parse_args()
+
+    client = TranscriptionClient(
+        args.server,
+        args.port,
+        lang=args.lang,
+        translate=args.translate,
+        model=args.model,                                  # also support hf_model => `Systran/faster-whisper-small`
+        use_vad=True,
+        save_output_recording=args.save_output_recording,  # Only used for microphone input, False by Default
+        output_recording_filename=args.output_file,        # Only used for microphone input
+        mute_audio_playback=args.mute_audio_playback,      # Only used for file input, False by Default
+        enable_translation=args.enable_translation,        # Enable translation of the transcription output
+        target_language=args.target_language,              # Target language for translation, e.g., "fr
+        enable_timestamps=args.enable_timestamps,
+        display_segments=args.n_display_segments,
+    )
+
+    if args.files is None:
+        client()
+        sys.exit(0)
+
+    # Validate audio files
+    valid_files = []
+    for file_path in args.files:
+        path = Path(file_path)
+        if path.exists() and path.is_file():
+            valid_files.append(str(path))
+        else:
+            print(f"Warning: File not found: {file_path}")
+
+    if not valid_files:
+        print("Error: No valid audio files found!")
+        sys.exit(1)
+
+    print(f"Found {len(valid_files)} audio file(s) to stream:")
+    for file_path in valid_files:
+        print(f"  - {file_path}")
+
+    for f in valid_files:
+        client(f)

run_server.py

@@ -1,5 +1,14 @@
 import argparse
-from whisper_live.server import TranscriptionServer
+import os
+import threading
+import logging
+from fastapi import FastAPI
+from fastapi import UploadFile, Form
+import uvicorn
+import tempfile
+import shutil
+import json
+from starlette.responses import PlainTextResponse, JSONResponse
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
@@ -10,7 +19,7 @@ if __name__ == "__main__":
     parser.add_argument('--backend', '-b',
                         type=str,
                         default='faster_whisper',
-                        help='Backends from ["tensorrt", "faster_whisper"]')
+                        help='Backends from ["tensorrt", "faster_whisper", "openvino"]')
     parser.add_argument('--faster_whisper_custom_model_path', '-fw',
                         type=str, default=None,
                         help="Custom Faster Whisper Model")
@@ -21,12 +30,70 @@ if __name__ == "__main__":
     parser.add_argument('--trt_multilingual', '-m',
                         action="store_true",
                         help='Boolean only for TensorRT model. True if multilingual.')
+    parser.add_argument('--trt_py_session',
+                        action="store_true",
+                        help='Boolean only for TensorRT model. Use python session or cpp session, By default uses Cpp.')
+    parser.add_argument('--omp_num_threads', '-omp',
+                        type=int,
+                        default=1,
+                        help="Number of threads to use for OpenMP")
+    parser.add_argument('--no_single_model', '-nsm',
+                        action='store_true',
+                        help='Set this if every connection should instantiate its own model. Only relevant for custom model, passed using -trt or -fw.')
+    parser.add_argument('--max_clients',
+                        type=int,
+                        default=4,
+                        help='Maximum clients supported by the server.')
+    parser.add_argument('--max_connection_time',
+                        type=int,
+                        default=300,
+                        help='The maximum duration (in seconds) a client can stay connected. Defaults to 300 seconds (5 minutes)')
+    parser.add_argument('--cache_path', '-c',
+                        type=str,
+                        default="~/.cache/whisper-live/",
+                        help='Path to cache the converted ctranslate2 models.')
+    parser.add_argument(
+        "--rest_port", type=int, default=8000, help="Port for the REST API server."
+    )
+    parser.add_argument(
+        "--enable_rest",
+        action="store_true",
+        help="Enable the OpenAI-compatible REST API endpoint.",
+    )
+    parser.add_argument(
+        '--cors-origins',
+        type=str,
+        default=None,
+        help="Comma-separated list of allowed CORS origins (e.g., 'http://localhost:3000,http://example.com'). Defaults to localhost/127.0.0.1 on the WebSocket port."
+    )
+    parser.add_argument(
+        '--batch_inference',
+        action='store_true',
+        help='Enable batched GPU inference for concurrent sessions. '
+             'Batches multiple sessions into a single GPU call for higher throughput.'
+    )
+    parser.add_argument(
+        '--batch_max_size',
+        type=int,
+        default=8,
+        help='Maximum batch size for batched inference (default: 8).'
+    )
+    parser.add_argument(
+        '--batch_window_ms',
+        type=int,
+        default=50,
+        help='Maximum time in ms to wait for batch to fill (default: 50).'
+    )
     args = parser.parse_args()
 
     if args.backend == "tensorrt":
         if args.trt_model_path is None:
             raise ValueError("Please Provide a valid tensorrt model path")
 
+    if "OMP_NUM_THREADS" not in os.environ:
+        os.environ["OMP_NUM_THREADS"] = str(args.omp_num_threads)
+
+    from whisper_live.server import TranscriptionServer
     server = TranscriptionServer()
     server.run(
         "0.0.0.0",
@@ -34,5 +101,16 @@ if __name__ == "__main__":
         backend=args.backend,
         faster_whisper_custom_model_path=args.faster_whisper_custom_model_path,
         whisper_tensorrt_path=args.trt_model_path,
-        trt_multilingual=args.trt_multilingual
-    )
+        trt_multilingual=args.trt_multilingual,
+        trt_py_session=args.trt_py_session,
+        single_model=not args.no_single_model,
+        max_clients=args.max_clients,
+        max_connection_time=args.max_connection_time,
+        cache_path=args.cache_path,
+        rest_port=args.rest_port,
+        enable_rest=args.enable_rest,
+        cors_origins=args.cors_origins,
+        batch_enabled=args.batch_inference,
+        batch_max_size=args.batch_max_size,
+        batch_window_ms=args.batch_window_ms,
+    )

scripts/build_whisper_tensorrt.sh

@@ -38,12 +38,24 @@ download_and_build_model() {
         "large-v3" | "large")
             model_url="https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt"
             ;;
+        "large-v3-turbo" | "turbo")
+            model_url="https://openaipublic.azureedge.net/main/whisper/models/aff26ae408abcba5fbf8813c21e62b0941638c5f6eebfb145be0c9839262a19a/large-v3-turbo.pt"
+            ;;
         *)
             echo "Invalid model name: $model_name"
             exit 1
             ;;
     esac
 
+    if [ "$model_name" == "turbo" ]; then
+        model_name="large-v3-turbo"
+    fi
+
+    local inference_precision="float16"
+    local weight_only_precision="${2:-float16}"
+    local max_beam_width=4
+    local max_batch_size=4
+
     echo "Downloading $model_name..."
     # wget --directory-prefix=assets "$model_url"
     # echo "Download completed: ${model_name}.pt"
@@ -54,11 +66,41 @@ download_and_build_model() {
         echo "${model_name}.pt already exists in assets directory."
     fi
 
-    local output_dir="whisper_${model_name//./_}"
+    local sanitized_model_name="${model_name//./_}"
+    local checkpoint_dir="whisper_${sanitized_model_name}_weights_${weight_only_precision}"
+    local output_dir="whisper_${sanitized_model_name}_${weight_only_precision}"
     echo "$output_dir"
-    echo "Running build script for $model_name with output directory $output_dir"
-    python3 build.py --output_dir "$output_dir" --use_gpt_attention_plugin --use_gemm_plugin --use_bert_attention_plugin --model_name "$model_name"
-    echo "Whisper $model_name TensorRT engine built."
+    echo "Converting model weights for $model_name..."
+    python3 convert_checkpoint.py \
+        $( [[ "$weight_only_precision" == "int8" || "$weight_only_precision" == "int4" ]] && echo "--use_weight_only --weight_only_precision $weight_only_precision" ) \
+        --output_dir "$checkpoint_dir" --model_name "$model_name"
+    
+    echo "Building encoder for $model_name..."
+    trtllm-build \
+        --checkpoint_dir "${checkpoint_dir}/encoder" \
+        --output_dir "${output_dir}/encoder" \
+        --moe_plugin disable \
+        --max_batch_size "$max_batch_size" \
+        --gemm_plugin disable \
+        --bert_attention_plugin "$inference_precision" \
+        --max_input_len 3000 \
+        --max_seq_len 3000
+    
+    echo "Building decoder for $model_name..."
+    trtllm-build \
+        --checkpoint_dir "${checkpoint_dir}/decoder" \
+        --output_dir "${output_dir}/decoder" \
+        --moe_plugin disable \
+        --max_beam_width "$max_beam_width" \
+        --max_batch_size "$max_batch_size" \
+        --max_seq_len 225 \
+        --max_input_len 32 \
+        --max_encoder_input_len 3000 \
+        --gemm_plugin "$inference_precision" \
+        --bert_attention_plugin "$inference_precision" \
+        --gpt_attention_plugin "$inference_precision"
+
+    echo "TensorRT LLM engine built for $model_name."
     echo "========================================="
     echo "Model is located at: $(pwd)/$output_dir"
 }
@@ -70,8 +112,9 @@ fi
 
 tensorrt_examples_dir="$1"
 model_name="${2:-small.en}"
+weight_only_precision="${3:-float16}"  # Default to float16 if not provided
 
-cd $1/whisper
+cd $tensorrt_examples_dir/whisper
 pip install --no-deps -r requirements.txt
 
-download_and_build_model "$model_name"
+download_and_build_model "$model_name" "$weight_only_precision"

scripts/setup.sh

@@ -1,3 +1,32 @@
-#! /bin/bash
+#!/bin/bash
 
-apt-get install portaudio19-dev ffmpeg wget -y
+# Detect the operating system
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # macOS
+    echo "Detected macOS, using Homebrew for installation"
+
+    # Check if Homebrew is installed
+    if ! command -v brew &> /dev/null; then
+        echo "Homebrew not found. Please install Homebrew first: https://brew.sh/"
+        exit 1
+    fi
+
+    # Install packages using Homebrew
+    brew install portaudio wget
+elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    # Linux
+    if [[ -f /etc/os-release ]]; then
+        source /etc/os-release
+    fi
+
+    if [[ "${ID:-}" == "fedora" ]]; then
+        echo "Detected Fedora, using dnf for installation"
+        dnf install -y portaudio-devel wget
+    else
+        echo "Detected Linux (assuming Debian/Ubuntu), using apt-get for installation"
+        apt-get install -y portaudio19-dev wget
+    fi
+else
+    echo "Unsupported operating system: $OSTYPE"
+    exit 1
+fi

setup.py

@@ -11,7 +11,7 @@ README = (HERE / "README.md").read_text()
 
 # This call to setup() does all the work
 setup(
-    name="whisper-live",
+    name="whisper_live",
     version=__version__,
     description="A nearly-live implementation of OpenAI's Whisper.",
     long_description=README,
@@ -43,18 +43,25 @@ setup(
     ),
     install_requires=[
         "PyAudio",
-        "faster-whisper==0.10.0",
+        "faster-whisper==1.2.0",
         "torch",
         "torchaudio",
         "websockets",
-        "onnxruntime==1.16.0",
-        "ffmpeg-python",
+        "onnxruntime==1.17.0",
         "scipy",
         "websocket-client",
         "numba",
-        "openai-whisper",
+        "openai-whisper==20250625",
         "kaldialign",
         "soundfile",
+        "tokenizers==0.20.3",
+        "librosa",
+        "numpy==1.26.4",
+        "openvino",
+        "openvino-genai",
+        "openvino-tokenizers",
+        "optimum", 
+        "optimum-intel",
     ],
-    python_requires=">=3.8"
+    python_requires=">=3.9"
 )

tests/test_batch_inference.py

@@ -0,0 +1,163 @@
+import time
+import unittest
+from unittest import mock
+from unittest.mock import MagicMock
+
+import numpy as np
+
+from whisper_live.batch_inference import BatchInferenceWorker, BatchRequest
+
+
+class TestBatchInferenceWorker(unittest.TestCase):
+    def setUp(self):
+        self.mock_transcriber = MagicMock()
+        self.worker = BatchInferenceWorker(
+            transcriber=self.mock_transcriber,
+            max_batch_size=8,
+            batch_window_ms=200,
+        )
+        self.worker.start()
+
+    def tearDown(self):
+        self.worker.stop()
+
+    def _make_audio(self, duration_s=1.0):
+        return np.random.randn(int(16000 * duration_s)).astype(np.float32)
+
+    def test_single_request_uses_transcribe(self):
+        """Single request should fall back to transcriber.transcribe()."""
+        fake_segment = MagicMock()
+        fake_info = MagicMock()
+        self.mock_transcriber.transcribe.return_value = ([fake_segment], fake_info)
+
+        req = BatchRequest(audio=self._make_audio(), language="en", use_vad=False)
+        self.worker.submit(req)
+        req.future.wait(timeout=5)
+
+        self.assertTrue(req.future.is_set())
+        self.assertIsNone(req.error)
+        self.assertEqual(req.result, [fake_segment])
+        self.assertEqual(req.info, fake_info)
+        self.mock_transcriber.transcribe.assert_called_once()
+
+    @mock.patch('whisper_live.batch_inference.get_suppressed_tokens', return_value=[-1])
+    @mock.patch('whisper_live.batch_inference.Tokenizer')
+    def test_multiple_requests_batched(self, mock_tokenizer_cls, mock_suppress):
+        """Multiple concurrent requests should go through the batched GPU path."""
+        # Mock tokenizer
+        mock_tok = MagicMock()
+        mock_tok.decode.return_value = "hello world"
+        mock_tokenizer_cls.return_value = mock_tok
+
+        # Mock feature extractor
+        self.mock_transcriber.feature_extractor.return_value = np.zeros(
+            (80, 3000), dtype=np.float32
+        )
+        self.mock_transcriber.feature_extractor.sampling_rate = 16000
+
+        # Mock encode
+        self.mock_transcriber.encode.return_value = np.zeros(
+            (3, 1500, 512), dtype=np.float32
+        )
+
+        # Mock model.generate — one result per item
+        gen_result = MagicMock()
+        gen_result.sequences_ids = [[50257, 50362, 1234, 50256]]
+        gen_result.scores = [np.float32(-1.0)]
+        gen_result.no_speech_prob = 0.1
+        self.mock_transcriber.model.generate.return_value = [gen_result] * 3
+
+        # Mock remaining model attributes
+        self.mock_transcriber.model.is_multilingual = False
+        self.mock_transcriber.max_length = 448
+        self.mock_transcriber.frames_per_second = 50
+        self.mock_transcriber.get_prompt.return_value = [50258]
+        self.mock_transcriber._split_segments_by_timestamps.return_value = (
+            [{"start": 0.0, "end": 1.0, "tokens": [1234], "seek": 0}],
+            None,
+            None,
+        )
+
+        requests = [
+            BatchRequest(audio=self._make_audio(), language="en", use_vad=False)
+            for _ in range(3)
+        ]
+        for req in requests:
+            self.worker.submit(req)
+        for req in requests:
+            req.future.wait(timeout=5)
+
+        for req in requests:
+            self.assertTrue(req.future.is_set())
+            self.assertIsNone(req.error)
+            self.assertIsNotNone(req.result)
+
+        # Verify the batched encode path was used (not transcribe)
+        self.mock_transcriber.encode.assert_called()
+        self.mock_transcriber.transcribe.assert_not_called()
+
+    def test_error_propagation(self):
+        """Transcriber errors should propagate to the request without crashing the worker."""
+        self.mock_transcriber.transcribe.side_effect = RuntimeError("GPU OOM")
+
+        req = BatchRequest(audio=self._make_audio(), language="en", use_vad=False)
+        self.worker.submit(req)
+        req.future.wait(timeout=5)
+
+        self.assertTrue(req.future.is_set())
+        self.assertIsInstance(req.error, RuntimeError)
+        self.assertIn("GPU OOM", str(req.error))
+
+        # Worker should still be alive — submit another request
+        self.mock_transcriber.transcribe.side_effect = None
+        self.mock_transcriber.transcribe.return_value = ([MagicMock()], MagicMock())
+
+        req2 = BatchRequest(audio=self._make_audio(), language="en", use_vad=False)
+        self.worker.submit(req2)
+        req2.future.wait(timeout=5)
+
+        self.assertIsNone(req2.error)
+        self.assertIsNotNone(req2.result)
+
+    def test_worker_stop(self):
+        """Worker thread should exit cleanly when stop() is called."""
+        self.assertTrue(self.worker._thread.is_alive())
+        self.worker.stop()
+        self.assertFalse(self.worker._thread.is_alive())
+
+    def test_batch_respects_max_size(self):
+        """Batches should not exceed max_batch_size."""
+        self.worker.stop()  # Stop the default worker
+
+        observed_batch_sizes = []
+        original_process = BatchInferenceWorker._process_batch
+
+        def tracking_process(self_inner, batch):
+            observed_batch_sizes.append(len(batch))
+            original_process(self_inner, batch)
+
+        self.worker = BatchInferenceWorker(
+            transcriber=self.mock_transcriber,
+            max_batch_size=2,
+            batch_window_ms=100,
+        )
+
+        self.mock_transcriber.transcribe.return_value = ([MagicMock()], MagicMock())
+
+        with mock.patch.object(
+            BatchInferenceWorker, '_process_batch', tracking_process
+        ):
+            self.worker.start()
+
+            requests = [
+                BatchRequest(audio=self._make_audio(), language="en", use_vad=False)
+                for _ in range(4)
+            ]
+            for req in requests:
+                self.worker.submit(req)
+            for req in requests:
+                req.future.wait(timeout=5)
+
+        for size in observed_batch_sizes:
+            self.assertLessEqual(size, 2)
+        self.assertTrue(all(req.future.is_set() for req in requests))

tests/test_client.py

@@ -2,10 +2,12 @@ import json
 import os
 import scipy
 import websocket
+import copy
 import unittest
 from unittest.mock import patch, MagicMock
-from whisper_live.client import TranscriptionClient
+from whisper_live.client import Client, TranscriptionClient, TranscriptionTeeClient
 from whisper_live.utils import resample
+from pathlib import Path
 
 
 class BaseTestCase(unittest.TestCase):
@@ -24,6 +26,7 @@ class BaseTestCase(unittest.TestCase):
 
         self.mock_pyaudio = mock_pyaudio
         self.mock_websocket = mock_websocket
+        self.mock_audio_packet = b'\x00\x01\x02\x03'
 
     def tearDown(self):
         self.client.close_websocket()
@@ -31,7 +34,6 @@ class BaseTestCase(unittest.TestCase):
         self.mock_websocket.stop()
         del self.client
 
-
 class TestClientWebSocketCommunication(BaseTestCase):
     def test_websocket_communication(self):
         expected_url = 'ws://localhost:9090'
@@ -46,7 +48,13 @@ class TestClientCallbacks(BaseTestCase):
             "language": self.client.language,
             "task": self.client.task,
             "model": self.client.model,
-            "use_vad": True
+            "use_vad": True,
+            "send_last_n_segments": 10,
+            "no_speech_thresh": 0.45,
+            "clip_audio": False,
+            "same_output_threshold": 10,
+            "enable_translation": False,
+            "target_language": "fr",
         })
         self.client.on_open(self.mock_ws_app)
         self.mock_ws_app.send.assert_called_with(expected_message)
@@ -64,15 +72,15 @@ class TestClientCallbacks(BaseTestCase):
         message = json.dumps({
             "uid": self.client.uid,
             "segments": [
-                {"start": 0, "end": 1, "text": "Test transcript"},
-                {"start": 1, "end": 2, "text": "Test transcript 2"},
-                {"start": 2, "end": 3, "text": "Test transcript 3"}
+                {"start": 0, "end": 1, "text": "Test transcript", "completed": True},
+                {"start": 1, "end": 2, "text": "Test transcript 2", "completed": True},
+                {"start": 2, "end": 3, "text": "Test transcript 3", "completed": True}
             ]
         })
         self.client.on_message(self.mock_ws_app, message)
 
         # Assert that the transcript was updated correctly
-        self.assertEqual(len(self.client.transcript), 2)
+        self.assertEqual(len(self.client.transcript), 3)
         self.assertEqual(self.client.transcript[1]['text'], "Test transcript 2")
 
     def test_on_close(self):
@@ -106,6 +114,49 @@ class TestAudioResampling(unittest.TestCase):
 
 class TestSendingAudioPacket(BaseTestCase):
     def test_send_packet(self):
-        mock_audio_packet = b'\x00\x01\x02\x03'
-        self.client.send_packet_to_server(mock_audio_packet)
-        self.client.client_socket.send.assert_called_with(mock_audio_packet, websocket.ABNF.OPCODE_BINARY)
+        self.client.send_packet_to_server(self.mock_audio_packet)
+        self.client.client_socket.send.assert_called_with(self.mock_audio_packet, websocket.ABNF.OPCODE_BINARY)
+
+class TestTee(BaseTestCase):
+    @patch('whisper_live.client.websocket.WebSocketApp')
+    @patch('whisper_live.client.pyaudio.PyAudio')
+    def setUp(self, mock_audio, mock_websocket):
+        super().setUp()
+        self.client2 = Client(host='localhost', port=9090, lang="es", translate=False, srt_file_path="transcript.srt")
+        self.client3 = Client(host='localhost', port=9090, lang="es", translate=True, srt_file_path="translation.srt")
+        # need a separate mock for each websocket
+        self.client3.client_socket = copy.deepcopy(self.client3.client_socket)
+        self.tee = TranscriptionTeeClient([self.client2, self.client3])
+
+    def tearDown(self):
+        self.tee.close_all_clients()
+        del self.tee
+        super().tearDown()
+
+    def test_invalid_constructor(self):
+        with self.assertRaises(Exception) as context:
+            TranscriptionTeeClient([])
+
+    def test_multicast_unconditional(self):
+        self.tee.multicast_packet(self.mock_audio_packet, True)
+        for client in self.tee.clients:
+            client.client_socket.send.assert_called_with(self.mock_audio_packet, websocket.ABNF.OPCODE_BINARY)
+
+    def test_multicast_conditional(self):
+        self.client2.recording = False
+        self.client3.recording = True
+        self.tee.multicast_packet(self.mock_audio_packet, False)
+        self.client2.client_socket.send.assert_not_called()
+        self.client3.client_socket.send.assert_called_with(self.mock_audio_packet, websocket.ABNF.OPCODE_BINARY)
+
+    def test_close_all(self):
+        self.tee.close_all_clients()
+        for client in self.tee.clients:
+            client.client_socket.close.assert_called()
+
+    def test_write_all_srt(self):
+        for client in self.tee.clients:
+            client.server_backend = "faster_whisper"
+        self.tee.write_all_clients_srt()
+        self.assertTrue(Path("transcript.srt").is_file())
+        self.assertTrue(Path("translation.srt").is_file())

tests/test_server.py

@@ -5,17 +5,18 @@ import unittest
 from unittest import mock
 
 import numpy as np
-import evaluate
+import jiwer
 
 from websockets.exceptions import ConnectionClosed
-from whisper_live.server import TranscriptionServer
-from whisper_live.client import TranscriptionClient
+from whisper_live.server import TranscriptionServer, BackendType, ClientManager
+from whisper_live.client import Client, TranscriptionClient, TranscriptionTeeClient
 from whisper.normalizers import EnglishTextNormalizer
 
 
 class TestTranscriptionServerInitialization(unittest.TestCase):
     def test_initialization(self):
         server = TranscriptionServer()
+        server.client_manager = ClientManager(max_clients=4, max_connection_time=600)
         self.assertEqual(server.client_manager.max_clients, 4)
         self.assertEqual(server.client_manager.max_connection_time, 600)
         self.assertDictEqual(server.client_manager.clients, {})
@@ -25,6 +26,7 @@ class TestTranscriptionServerInitialization(unittest.TestCase):
 class TestGetWaitTime(unittest.TestCase):
     def setUp(self):
         self.server = TranscriptionServer()
+        self.server.client_manager = ClientManager(max_clients=4, max_connection_time=600)
         self.server.client_manager.start_times = {
             'client1': time.time() - 120,
             'client2': time.time() - 300
@@ -40,6 +42,8 @@ class TestGetWaitTime(unittest.TestCase):
 class TestServerConnection(unittest.TestCase):
     def setUp(self):
         self.server = TranscriptionServer()
+        self.server.client_manager = ClientManager(max_clients=4, max_connection_time=600)
+        self.server.cache_path = "~/.cache/whisper-live/"
 
     @mock.patch('websockets.WebSocketCommonProtocol')
     def test_connection(self, mock_websocket):
@@ -49,7 +53,7 @@ class TestServerConnection(unittest.TestCase):
             'task': 'transcribe',
             'model': 'tiny.en'
         })
-        self.server.recv_audio(mock_websocket, "faster_whisper")
+        self.server.recv_audio(mock_websocket, BackendType("faster_whisper"))
 
     @mock.patch('websockets.WebSocketCommonProtocol')
     def test_recv_audio_exception_handling(self, mock_websocket):
@@ -61,7 +65,7 @@ class TestServerConnection(unittest.TestCase):
         }),  np.array([1, 2, 3]).tobytes()]
 
         with self.assertLogs(level="ERROR"):
-            self.server.recv_audio(mock_websocket, "faster_whisper")
+            self.server.recv_audio(mock_websocket, BackendType("faster_whisper"))
 
         self.assertNotIn(mock_websocket, self.server.client_manager.clients)
 
@@ -69,6 +73,10 @@ class TestServerConnection(unittest.TestCase):
 class TestServerInferenceAccuracy(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
+        cls.mock_pyaudio_patch = mock.patch('pyaudio.PyAudio')
+        cls.mock_pyaudio = cls.mock_pyaudio_patch.start()
+        cls.mock_pyaudio.return_value.open.return_value = mock.MagicMock()
+        
         cls.server_process = subprocess.Popen(["python", "run_server.py"])
         time.sleep(2)
 
@@ -77,32 +85,37 @@ class TestServerInferenceAccuracy(unittest.TestCase):
         cls.server_process.terminate()
         cls.server_process.wait()
 
-    @mock.patch('pyaudio.PyAudio')
-    def setUp(self, mock_pyaudio):
-        self.mock_pyaudio = mock_pyaudio.return_value
-        self.mock_stream = mock.MagicMock()
-        self.mock_pyaudio.open.return_value = self.mock_stream
-        self.metric = evaluate.load("wer")
+    def setUp(self):
         self.normalizer = EnglishTextNormalizer()
-        self.client = TranscriptionClient(
-            "localhost", "9090", model="base.en", lang="en",
-        )
 
-    def test_inference(self):
+    def check_prediction(self, srt_path):
         gt = "And so my fellow Americans, ask not, what your country can do for you. Ask what you can do for your country!"
-        self.client("assets/jfk.flac")
-        with open("output.srt", "r") as f:
+        with open(srt_path, "r") as f:
             lines = f.readlines()
             prediction = " ".join([line.strip() for line in lines[2::4]])
         prediction_normalized = self.normalizer(prediction)
         gt_normalized = self.normalizer(gt)
 
         # calculate WER
-        wer = self.metric.compute(
-            predictions=[prediction_normalized],
-            references=[gt_normalized]
+        wer_score = jiwer.wer(gt_normalized, prediction_normalized)
+        self.assertLess(wer_score, 0.05)
+
+    def test_inference(self):
+        client = TranscriptionClient(
+            "localhost", "9090", model="base.en", lang="en",
         )
-        self.assertLess(wer, 0.05)
+        client("assets/jfk.flac")
+        self.check_prediction("output.srt")
+
+    def test_simultaneous_inference(self):
+        client1 = Client(
+            "localhost", "9090", model="base.en", lang="en", srt_file_path="transcript1.srt")
+        client2 = Client(
+            "localhost", "9090", model="base.en", lang="en", srt_file_path="transcript2.srt")
+        tee = TranscriptionTeeClient([client1, client2])
+        tee("assets/jfk.flac")
+        self.check_prediction("transcript1.srt")
+        self.check_prediction("transcript2.srt")
 
 
 class TestExceptionHandling(unittest.TestCase):
@@ -111,10 +124,10 @@ class TestExceptionHandling(unittest.TestCase):
 
     @mock.patch('websockets.WebSocketCommonProtocol')
     def test_connection_closed_exception(self, mock_websocket):
-        mock_websocket.recv.side_effect = ConnectionClosed(1001, "testing connection closed")
+        mock_websocket.recv.side_effect = ConnectionClosed(1001, "testing connection closed", rcvd_then_sent=mock.Mock())
 
         with self.assertLogs(level="INFO") as log:
-            self.server.recv_audio(mock_websocket, "faster_whisper")
+            self.server.recv_audio(mock_websocket, BackendType("faster_whisper"))
             self.assertTrue(any("Connection closed by client" in message for message in log.output))
 
     @mock.patch('websockets.WebSocketCommonProtocol')
@@ -122,7 +135,7 @@ class TestExceptionHandling(unittest.TestCase):
         mock_websocket.recv.return_value = "invalid json"
 
         with self.assertLogs(level="ERROR") as log:
-            self.server.recv_audio(mock_websocket, "faster_whisper")
+            self.server.recv_audio(mock_websocket, BackendType("faster_whisper"))
             self.assertTrue(any("Failed to decode JSON from client" in message for message in log.output))
 
     @mock.patch('websockets.WebSocketCommonProtocol')
@@ -130,7 +143,7 @@ class TestExceptionHandling(unittest.TestCase):
         mock_websocket.recv.side_effect = RuntimeError("Unexpected error")
 
         with self.assertLogs(level="ERROR") as log:
-            self.server.recv_audio(mock_websocket, "faster_whisper")
+            self.server.recv_audio(mock_websocket, BackendType("faster_whisper"))
             for message in log.output:
                 print(message)
             print()

tests/test_vad.py

@@ -1,6 +1,6 @@
 import unittest
 import numpy as np
-from whisper_live.tensorrt_utils import load_audio
+from whisper_live.transcriber.tensorrt_utils import load_audio
 from whisper_live.vad import VoiceActivityDetector
 
 

whisper_live/__init__.py

@@ -0,0 +1,3 @@
+from whisper_live.__version__ import __version__
+
+__all__ = ['__version__']

whisper_live/__version__.py

@@ -1 +1 @@
-__version__ = "0.2.1"
+__version__ = "0.8.0"

whisper_live/backend/base.py

@@ -0,0 +1,379 @@
+import json
+import logging
+import threading
+import time
+import queue
+import numpy as np
+
+
+class ServeClientBase(object):
+    RATE = 16000
+    SERVER_READY = "SERVER_READY"
+    DISCONNECT = "DISCONNECT"
+
+    client_uid: str
+    """A unique identifier for the client."""
+    websocket: object
+    """The WebSocket connection for the client."""
+    send_last_n_segments: int
+    """Number of most recent segments to send to the client."""
+    no_speech_thresh: float
+    """Segments with no speech probability above this threshold will be discarded."""
+    clip_audio: bool
+    """Whether to clip audio with no valid segments."""
+    same_output_threshold: int
+    """Number of repeated outputs before considering it as a valid segment."""
+
+    def __init__(
+        self,
+        client_uid,
+        websocket,
+        send_last_n_segments=10,
+        no_speech_thresh=0.45,
+        clip_audio=False,
+        same_output_threshold=10,
+        translation_queue=None,
+    ):
+        self.client_uid = client_uid
+        self.websocket = websocket
+        self.send_last_n_segments = send_last_n_segments
+        self.no_speech_thresh = no_speech_thresh
+        self.clip_audio = clip_audio
+        self.same_output_threshold = same_output_threshold
+
+        self.frames = b""
+        self.timestamp_offset = 0.0
+        self.frames_np = None
+        self.frames_offset = 0.0
+        self.text = []
+        self.current_out = ""
+        self.prev_out = ""
+        self.exit = False
+        self.same_output_count = 0
+        self.transcript = []
+        self.end_time_for_same_output = None
+        self.translation_queue = translation_queue
+
+        # threading
+        self.lock = threading.Lock()
+
+    def speech_to_text(self):
+        """
+        Process an audio stream in an infinite loop, continuously transcribing the speech.
+
+        This method continuously receives audio frames, performs real-time transcription, and sends
+        transcribed segments to the client via a WebSocket connection.
+
+        If the client's language is not detected, it waits for 30 seconds of audio input to make a language prediction.
+        It utilizes the Whisper ASR model to transcribe the audio, continuously processing and streaming results. Segments
+        are sent to the client in real-time, and a history of segments is maintained to provide context.
+
+        Raises:
+            Exception: If there is an issue with audio processing or WebSocket communication.
+
+        """
+        while True:
+            if self.exit:
+                logging.info("Exiting speech to text thread")
+                break
+
+            if self.frames_np is None:
+                continue
+
+            if self.clip_audio:
+                self.clip_audio_if_no_valid_segment()
+
+            input_bytes, duration = self.get_audio_chunk_for_processing()
+            if duration < 1.0:
+                time.sleep(0.1)     # wait for audio chunks to arrive
+                continue
+            try:
+                input_sample = input_bytes.copy()
+                result = self.transcribe_audio(input_sample)
+
+                if result is None or self.language is None:
+                    self.timestamp_offset += duration
+                    time.sleep(0.25)    # wait for voice activity, result is None when no voice activity
+                    continue
+                self.handle_transcription_output(result, duration)
+
+            except Exception as e:
+                logging.error(f"[ERROR]: Failed to transcribe audio chunk: {e}")
+                time.sleep(0.01)
+
+    def transcribe_audio(self):
+        raise NotImplementedError
+
+    def handle_transcription_output(self, result, duration):
+        raise NotImplementedError
+    
+    def format_segment(self, start, end, text, completed=False):
+        """
+        Formats a transcription segment with precise start and end times alongside the transcribed text.
+
+        Args:
+            start (float): The start time of the transcription segment in seconds.
+            end (float): The end time of the transcription segment in seconds.
+            text (str): The transcribed text corresponding to the segment.
+
+        Returns:
+            dict: A dictionary representing the formatted transcription segment, including
+                'start' and 'end' times as strings with three decimal places and the 'text'
+                of the transcription.
+        """
+        return {
+            'start': "{:.3f}".format(start),
+            'end': "{:.3f}".format(end),
+            'text': text,
+            'completed': completed
+        }
+
+    def add_frames(self, frame_np):
+        """
+        Add audio frames to the ongoing audio stream buffer.
+
+        This method is responsible for maintaining the audio stream buffer, allowing the continuous addition
+        of audio frames as they are received. It also ensures that the buffer does not exceed a specified size
+        to prevent excessive memory usage.
+
+        If the buffer size exceeds a threshold (45 seconds of audio data), it discards the oldest 30 seconds
+        of audio data to maintain a reasonable buffer size. If the buffer is empty, it initializes it with the provided
+        audio frame. The audio stream buffer is used for real-time processing of audio data for transcription.
+
+        Args:
+            frame_np (numpy.ndarray): The audio frame data as a NumPy array.
+
+        """
+        self.lock.acquire()
+        if self.frames_np is not None and self.frames_np.shape[0] > 45*self.RATE:
+            self.frames_offset += 30.0
+            self.frames_np = self.frames_np[int(30*self.RATE):]
+            # check timestamp offset(should be >= self.frame_offset)
+            # this basically means that there is no speech as timestamp offset hasnt updated
+            # and is less than frame_offset
+            if self.timestamp_offset < self.frames_offset:
+                self.timestamp_offset = self.frames_offset
+        if self.frames_np is None:
+            self.frames_np = frame_np.copy()
+        else:
+            self.frames_np = np.concatenate((self.frames_np, frame_np), axis=0)
+        self.lock.release()
+
+    def clip_audio_if_no_valid_segment(self):
+        """
+        Update the timestamp offset based on audio buffer status.
+        Clip audio if the current chunk exceeds 30 seconds, this basically implies that
+        no valid segment for the last 30 seconds from whisper
+        """
+        with self.lock:
+            if self.frames_np[int((self.timestamp_offset - self.frames_offset)*self.RATE):].shape[0] > 25 * self.RATE:
+                duration = self.frames_np.shape[0] / self.RATE
+                self.timestamp_offset = self.frames_offset + duration - 5
+
+    def get_audio_chunk_for_processing(self):
+        """
+        Retrieves the next chunk of audio data for processing based on the current offsets.
+
+        Calculates which part of the audio data should be processed next, based on
+        the difference between the current timestamp offset and the frame's offset, scaled by
+        the audio sample rate (RATE). It then returns this chunk of audio data along with its
+        duration in seconds.
+
+        Returns:
+            tuple: A tuple containing:
+                - input_bytes (np.ndarray): The next chunk of audio data to be processed.
+                - duration (float): The duration of the audio chunk in seconds.
+        """
+        with self.lock:
+            samples_take = max(0, (self.timestamp_offset - self.frames_offset) * self.RATE)
+            input_bytes = self.frames_np[int(samples_take):].copy()
+        duration = input_bytes.shape[0] / self.RATE
+        return input_bytes, duration
+
+    def prepare_segments(self, last_segment=None):
+        """
+        Prepares the segments of transcribed text to be sent to the client.
+
+        This method compiles the recent segments of transcribed text, ensuring that only the
+        specified number of the most recent segments are included. It also appends the most
+        recent segment of text if provided (which is considered incomplete because of the possibility
+        of the last word being truncated in the audio chunk).
+
+        Args:
+            last_segment (str, optional): The most recent segment of transcribed text to be added
+                                          to the list of segments. Defaults to None.
+
+        Returns:
+            list: A list of transcribed text segments to be sent to the client.
+        """
+        segments = []
+        if len(self.transcript) >= self.send_last_n_segments:
+            segments = self.transcript[-self.send_last_n_segments:].copy()
+        else:
+            segments = self.transcript.copy()
+        if last_segment is not None:
+            segments = segments + [last_segment]
+        return segments
+
+    def get_audio_chunk_duration(self, input_bytes):
+        """
+        Calculates the duration of the provided audio chunk.
+
+        Args:
+            input_bytes (numpy.ndarray): The audio chunk for which to calculate the duration.
+
+        Returns:
+            float: The duration of the audio chunk in seconds.
+        """
+        return input_bytes.shape[0] / self.RATE
+
+    def send_transcription_to_client(self, segments):
+        """
+        Sends the specified transcription segments to the client over the websocket connection.
+
+        This method formats the transcription segments into a JSON object and attempts to send
+        this object to the client. If an error occurs during the send operation, it logs the error.
+
+        Returns:
+            segments (list): A list of transcription segments to be sent to the client.
+        """
+        try:
+            self.websocket.send(
+                json.dumps({
+                    "uid": self.client_uid,
+                    "segments": segments,
+                })
+            )
+        except Exception as e:
+            logging.error(f"[ERROR]: Sending data to client: {e}")
+
+    def disconnect(self):
+        """
+        Notify the client of disconnection and send a disconnect message.
+
+        This method sends a disconnect message to the client via the WebSocket connection to notify them
+        that the transcription service is disconnecting gracefully.
+
+        """
+        self.websocket.send(json.dumps({
+            "uid": self.client_uid,
+            "message": self.DISCONNECT
+        }))
+
+    def cleanup(self):
+        """
+        Perform cleanup tasks before exiting the transcription service.
+
+        This method performs necessary cleanup tasks, including stopping the transcription thread, marking
+        the exit flag to indicate the transcription thread should exit gracefully, and destroying resources
+        associated with the transcription process.
+
+        """
+        logging.info("Cleaning up.")
+        self.exit = True
+    
+    def get_segment_no_speech_prob(self, segment):
+        return getattr(segment, "no_speech_prob", 0)
+
+    def get_segment_start(self, segment):
+        return getattr(segment, "start", getattr(segment, "start_ts", 0))
+
+    def get_segment_end(self, segment):
+        return getattr(segment, "end", getattr(segment, "end_ts", 0))
+
+    def update_segments(self, segments, duration):
+        """
+        Processes the segments from Whisper and updates the transcript.
+        Uses helper methods to account for differences between backends.
+        
+        Args:
+            segments (list): List of segments returned by the transcriber.
+            duration (float): Duration of the current audio chunk.
+        
+        Returns:
+            dict or None: The last processed segment (if any).
+        """
+        offset = None
+        self.current_out = ''
+        last_segment = None
+
+        # Process complete segments only if there are more than one
+        # and if the last segment's no_speech_prob is below the threshold.
+        if len(segments) > 1 and self.get_segment_no_speech_prob(segments[-1]) <= self.no_speech_thresh:
+            for s in segments[:-1]:
+                text_ = s.text
+                self.text.append(text_)
+                with self.lock:
+                    start = self.timestamp_offset + self.get_segment_start(s)
+                    end = self.timestamp_offset + min(duration, self.get_segment_end(s))
+                if start >= end:
+                    continue
+                if self.get_segment_no_speech_prob(s) > self.no_speech_thresh:
+                    continue
+                completed_segment = self.format_segment(start, end, text_, completed=True)
+                self.transcript.append(completed_segment)
+
+                if self.translation_queue:
+                    try:
+                        self.translation_queue.put(completed_segment.copy(), timeout=0.1)
+                    except queue.Full:
+                        logging.warning("Translation queue is full, skipping segment")
+                offset = min(duration, self.get_segment_end(s))
+
+        # Process the last segment if its no_speech_prob is acceptable.
+        if self.get_segment_no_speech_prob(segments[-1]) <= self.no_speech_thresh:
+            self.current_out += segments[-1].text
+            with self.lock:
+                last_segment = self.format_segment(
+                    self.timestamp_offset + self.get_segment_start(segments[-1]),
+                    self.timestamp_offset + min(duration, self.get_segment_end(segments[-1])),
+                    self.current_out,
+                    completed=False
+                )
+
+        # Handle repeated output logic.
+        if self.current_out.strip() == self.prev_out.strip() and self.current_out != '':
+            self.same_output_count += 1
+
+            # if we remove the audio because of same output on the nth reptition we might remove the 
+            # audio thats not yet transcribed so, capturing the time when it was repeated for the first time
+            if self.end_time_for_same_output is None:
+                self.end_time_for_same_output = self.get_segment_end(segments[-1])
+            time.sleep(0.1)  # wait briefly for any new voice activity
+        else:
+            self.same_output_count = 0
+            self.end_time_for_same_output = None
+
+        # If the same incomplete segment is repeated too many times,
+        # append it to the transcript and update the offset.
+        if self.same_output_count > self.same_output_threshold:
+            if not self.text or self.text[-1].strip().lower() != self.current_out.strip().lower():
+                self.text.append(self.current_out)
+                with self.lock:
+                    completed_segment = self.format_segment(
+                        self.timestamp_offset,
+                        self.timestamp_offset + min(duration, self.end_time_for_same_output),
+                        self.current_out,
+                        completed=True
+                    )
+                    self.transcript.append(completed_segment)
+
+                    if self.translation_queue:
+                        try:
+                            self.translation_queue.put(completed_segment.copy(), timeout=0.1)
+                        except queue.Full:
+                            logging.warning("Translation queue is full, skipping segment")
+
+            self.current_out = ''
+            offset = min(duration, self.end_time_for_same_output)
+            self.same_output_count = 0
+            last_segment = None
+            self.end_time_for_same_output = None
+        else:
+            self.prev_out = self.current_out
+
+        if offset is not None:
+            with self.lock:
+                self.timestamp_offset += offset
+
+        return last_segment

whisper_live/backend/faster_whisper_backend.py

@@ -0,0 +1,257 @@
+import os
+import json
+import logging
+import threading
+import time
+import torch
+import ctranslate2
+from huggingface_hub import snapshot_download
+
+from whisper_live.transcriber.transcriber_faster_whisper import WhisperModel
+from whisper_live.backend.base import ServeClientBase
+
+
+class ServeClientFasterWhisper(ServeClientBase):
+    SINGLE_MODEL = None
+    SINGLE_MODEL_LOCK = threading.Lock()
+    BATCH_WORKER = None
+
+    def __init__(
+        self,
+        websocket,
+        task="transcribe",
+        device=None,
+        language=None,
+        client_uid=None,
+        model="small.en",
+        initial_prompt=None,
+        vad_parameters=None,
+        use_vad=True,
+        single_model=False,
+        send_last_n_segments=10,
+        no_speech_thresh=0.45,
+        clip_audio=False,
+        same_output_threshold=7,
+        cache_path="~/.cache/whisper-live/",
+        translation_queue=None,
+    ):
+        """
+        Initialize a ServeClient instance.
+        The Whisper model is initialized based on the client's language and device availability.
+        The transcription thread is started upon initialization. A "SERVER_READY" message is sent
+        to the client to indicate that the server is ready.
+
+        Args:
+            websocket (WebSocket): The WebSocket connection for the client.
+            task (str, optional): The task type, e.g., "transcribe". Defaults to "transcribe".
+            device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None.
+            language (str, optional): The language for transcription. Defaults to None.
+            client_uid (str, optional): A unique identifier for the client. Defaults to None.
+            model (str, optional): The whisper model size. Defaults to 'small.en'
+            initial_prompt (str, optional): Prompt for whisper inference. Defaults to None.
+            single_model (bool, optional): Whether to instantiate a new model for each client connection. Defaults to False.
+            send_last_n_segments (int, optional): Number of most recent segments to send to the client. Defaults to 10.
+            no_speech_thresh (float, optional): Segments with no speech probability above this threshold will be discarded. Defaults to 0.45.
+            clip_audio (bool, optional): Whether to clip audio with no valid segments. Defaults to False.
+            same_output_threshold (int, optional): Number of repeated outputs before considering it as a valid segment. Defaults to 10.
+
+        """
+        super().__init__(
+            client_uid,
+            websocket,
+            send_last_n_segments,
+            no_speech_thresh,
+            clip_audio,
+            same_output_threshold,
+            translation_queue
+        )
+        self.cache_path = cache_path
+        self.model_sizes = [
+            "tiny", "tiny.en", "base", "base.en", "small", "small.en",
+            "medium", "medium.en", "large-v2", "large-v3", "distil-small.en",
+            "distil-medium.en", "distil-large-v2", "distil-large-v3",
+            "large-v3-turbo", "turbo"
+        ]
+
+        self.model_size_or_path = model
+        self.language = "en" if self.model_size_or_path.endswith("en") else language
+        self.task = task
+        self.initial_prompt = initial_prompt
+        self.vad_parameters = vad_parameters or {"threshold": 0.5}
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        if device == "cuda":
+            major, _ = torch.cuda.get_device_capability(device)
+            self.compute_type = "float16" if major >= 7 else "float32"
+        else:
+            self.compute_type = "int8"
+
+        if self.model_size_or_path is None:
+            return
+        logging.info(f"Using Device={device} with precision {self.compute_type}")
+    
+        try:
+            if single_model:
+                if ServeClientFasterWhisper.SINGLE_MODEL is None:
+                    self.create_model(device)
+                    ServeClientFasterWhisper.SINGLE_MODEL = self.transcriber
+                else:
+                    self.transcriber = ServeClientFasterWhisper.SINGLE_MODEL
+            else:
+                self.create_model(device)
+        except Exception as e:
+            logging.error(f"Failed to load model: {e}")
+            self.websocket.send(json.dumps({
+                "uid": self.client_uid,
+                "status": "ERROR",
+                "message": f"Failed to load model: {str(self.model_size_or_path)}"
+            }))
+            self.websocket.close()
+            return
+
+        self.use_vad = use_vad
+
+        # threading
+        self.trans_thread = threading.Thread(target=self.speech_to_text)
+        self.trans_thread.start()
+        self.websocket.send(
+            json.dumps(
+                {
+                    "uid": self.client_uid,
+                    "message": self.SERVER_READY,
+                    "backend": "faster_whisper"
+                }
+            )
+        )
+
+    def create_model(self, device):
+        """
+        Instantiates a new model, sets it as the transcriber. If model is a huggingface model_id
+        then it is automatically converted to ctranslate2(faster_whisper) format.
+        """
+        model_ref = self.model_size_or_path
+
+        if model_ref in self.model_sizes:
+            model_to_load = model_ref
+        else:
+            logging.info(f"Model not in model_sizes")
+            if os.path.isdir(model_ref) and ctranslate2.contains_model(model_ref):
+                model_to_load = model_ref
+            else:
+                local_snapshot = snapshot_download(
+                    repo_id = model_ref,
+                    repo_type = "model",
+                )
+                if ctranslate2.contains_model(local_snapshot):
+                    model_to_load = local_snapshot
+                else:
+                    cache_root = os.path.expanduser(os.path.join(self.cache_path, "whisper-ct2-models/"))
+                    os.makedirs(cache_root, exist_ok=True)
+                    safe_name = model_ref.replace("/", "--")
+                    ct2_dir = os.path.join(cache_root, safe_name)
+
+                    if not ctranslate2.contains_model(ct2_dir):
+                        logging.info(f"Converting '{model_ref}' to CTranslate2 @ {ct2_dir}")
+                        ct2_converter = ctranslate2.converters.TransformersConverter(
+                            local_snapshot, 
+                            copy_files=["tokenizer.json", "preprocessor_config.json"]
+                        )
+                        ct2_converter.convert(
+                            output_dir=ct2_dir,
+                            quantization=self.compute_type,
+                            force=False,  # skip if already up-to-date
+                        )
+                    model_to_load = ct2_dir
+
+        logging.info(f"Loading model: {model_to_load}")
+        self.transcriber = WhisperModel(
+            model_to_load,
+            device=device,
+            compute_type=self.compute_type,
+            local_files_only=False,
+        )
+
+    def set_language(self, info):
+        """
+        Updates the language attribute based on the detected language information.
+
+        Args:
+            info (object): An object containing the detected language and its probability. This object
+                        must have at least two attributes: `language`, a string indicating the detected
+                        language, and `language_probability`, a float representing the confidence level
+                        of the language detection.
+        """
+        if info.language_probability > 0.5:
+            self.language = info.language
+            logging.info(f"Detected language {self.language} with probability {info.language_probability}")
+            self.websocket.send(json.dumps(
+                {"uid": self.client_uid, "language": self.language, "language_prob": info.language_probability}))
+
+    def transcribe_audio(self, input_sample):
+        """
+        Transcribes the provided audio sample using the configured transcriber instance.
+
+        If the language has not been set, it updates the session's language based on the transcription
+        information.
+
+        Args:
+            input_sample (np.array): The audio chunk to be transcribed. This should be a NumPy
+                                    array representing the audio data.
+
+        Returns:
+            The transcription result from the transcriber. The exact format of this result
+            depends on the implementation of the `transcriber.transcribe` method but typically
+            includes the transcribed text.
+        """
+        # Batch inference path: submit to central queue and wait
+        if ServeClientFasterWhisper.BATCH_WORKER is not None:
+            from whisper_live.batch_inference import BatchRequest
+            request = BatchRequest(
+                audio=input_sample,
+                language=self.language,
+                task=self.task,
+                initial_prompt=self.initial_prompt,
+                use_vad=self.use_vad,
+                vad_parameters=self.vad_parameters if self.use_vad else None,
+            )
+            ServeClientFasterWhisper.BATCH_WORKER.submit(request)
+            request.future.wait(timeout=30)
+            if request.error:
+                raise request.error
+            if self.language is None and request.info is not None:
+                self.set_language(request.info)
+            return request.result
+
+        # Original lock-based path (backward compatible)
+        if ServeClientFasterWhisper.SINGLE_MODEL:
+            ServeClientFasterWhisper.SINGLE_MODEL_LOCK.acquire()
+        result, info = self.transcriber.transcribe(
+            input_sample,
+            initial_prompt=self.initial_prompt,
+            language=self.language,
+            task=self.task,
+            vad_filter=self.use_vad,
+            vad_parameters=self.vad_parameters if self.use_vad else None)
+        if ServeClientFasterWhisper.SINGLE_MODEL:
+            ServeClientFasterWhisper.SINGLE_MODEL_LOCK.release()
+
+        if self.language is None and info is not None:
+            self.set_language(info)
+        return result
+
+    def handle_transcription_output(self, result, duration):
+        """
+        Handle the transcription output, updating the transcript and sending data to the client.
+
+        Args:
+            result (str): The result from whisper inference i.e. the list of segments.
+            duration (float): Duration of the transcribed audio chunk.
+        """
+        segments = []
+        if len(result):
+            self.t_start = None
+            last_segment = self.update_segments(result, duration)
+            segments = self.prepare_segments(last_segment)
+
+        if len(segments):
+            self.send_transcription_to_client(segments)

whisper_live/backend/openvino_backend.py

@@ -0,0 +1,148 @@
+import json
+import logging
+import threading
+import time
+
+from openvino import Core
+from whisper_live.backend.base import ServeClientBase
+from whisper_live.transcriber.transcriber_openvino import WhisperOpenVINO
+
+
+class ServeClientOpenVINO(ServeClientBase):
+    SINGLE_MODEL = None
+    SINGLE_MODEL_LOCK = threading.Lock()
+
+    def __init__(
+        self,
+        websocket,
+        task="transcribe",
+        device=None,
+        language=None,
+        client_uid=None,
+        model="small.en",
+        initial_prompt=None,
+        vad_parameters=None,
+        use_vad=True,
+        single_model=False,
+        send_last_n_segments=10,
+        no_speech_thresh=0.45,
+        clip_audio=False,
+        same_output_threshold=10,
+    ):
+        """
+        Initialize a ServeClient instance.
+        The Whisper model is initialized based on the client's language and device availability.
+        The transcription thread is started upon initialization. A "SERVER_READY" message is sent
+        to the client to indicate that the server is ready.
+
+        Args:
+            websocket (WebSocket): The WebSocket connection for the client.
+            task (str, optional): The task type, e.g., "transcribe." Defaults to "transcribe".
+            device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None.
+            language (str, optional): The language for transcription. Defaults to None.
+            client_uid (str, optional): A unique identifier for the client. Defaults to None.
+            model (str, optional): Huggingface model_id for a valid OpenVINO model.
+            initial_prompt (str, optional): Prompt for whisper inference. Defaults to None.
+            single_model (bool, optional): Whether to instantiate a new model for each client connection. Defaults to False.
+            send_last_n_segments (int, optional): Number of most recent segments to send to the client. Defaults to 10.
+            no_speech_thresh (float, optional): Segments with no speech probability above this threshold will be discarded. Defaults to 0.45.
+            clip_audio (bool, optional): Whether to clip audio with no valid segments. Defaults to False.
+            same_output_threshold (int, optional): Number of repeated outputs before considering it as a valid segment. Defaults to 10.
+        """
+        super().__init__(
+            client_uid,
+            websocket,
+            send_last_n_segments,
+            no_speech_thresh,
+            clip_audio,
+            same_output_threshold,
+        )
+        self.language = "en" if language is None else language
+        if not self.language.startswith("<|"):
+            self.language = f"<|{self.language}|>"
+
+        self.task = "transcribe" if task is None else task
+
+        self.clip_audio = True
+
+        core = Core()
+        available_devices = core.available_devices
+        if 'GPU' in available_devices:
+            selected_device = 'GPU'
+        else:
+            gpu_devices = [d for d in available_devices if d.startswith('GPU')]
+            selected_device = gpu_devices[0] if gpu_devices else 'CPU'
+        self.device = selected_device
+
+
+        if single_model:
+            if ServeClientOpenVINO.SINGLE_MODEL is None:
+                self.create_model(model)
+                ServeClientOpenVINO.SINGLE_MODEL = self.transcriber
+            else:
+                self.transcriber = ServeClientOpenVINO.SINGLE_MODEL
+        else:
+            self.create_model(model)
+
+        # threading
+        self.trans_thread = threading.Thread(target=self.speech_to_text)
+        self.trans_thread.start()
+
+        self.websocket.send(json.dumps({
+            "uid": self.client_uid,
+            "message": self.SERVER_READY,
+            "backend": "openvino"
+        }))
+        logging.info(f"Using OpenVINO device: {self.device}")
+        logging.info(f"Running OpenVINO backend with language: {self.language} and task: {self.task}")
+
+    def create_model(self, model_id):
+        """
+        Instantiates a new model, sets it as the transcriber.
+        """
+        self.transcriber = WhisperOpenVINO(
+            model_id,
+            device=self.device,
+            language=self.language,
+            task=self.task
+        )
+
+    def transcribe_audio(self, input_sample):
+        """
+        Transcribes the provided audio sample using the configured transcriber instance.
+
+        If the language has not been set, it updates the session's language based on the transcription
+        information.
+
+        Args:
+            input_sample (np.array): The audio chunk to be transcribed. This should be a NumPy
+                                    array representing the audio data.
+
+        Returns:
+            The transcription result from the transcriber. The exact format of this result
+            depends on the implementation of the `transcriber.transcribe` method but typically
+            includes the transcribed text.
+        """
+        if ServeClientOpenVINO.SINGLE_MODEL:
+            ServeClientOpenVINO.SINGLE_MODEL_LOCK.acquire()
+        result = self.transcriber.transcribe(input_sample)
+        if ServeClientOpenVINO.SINGLE_MODEL:
+            ServeClientOpenVINO.SINGLE_MODEL_LOCK.release()
+        return result
+
+    def handle_transcription_output(self, result, duration):
+        """
+        Handle the transcription output, updating the transcript and sending data to the client.
+
+        Args:
+            result (str): The result from whisper inference i.e. the list of segments.
+            duration (float): Duration of the transcribed audio chunk.
+        """
+        segments = []
+        if len(result):
+            self.t_start = None
+            last_segment = self.update_segments(result, duration)
+            segments = self.prepare_segments(last_segment)
+
+        if len(segments):
+            self.send_transcription_to_client(segments)

whisper_live/backend/tokenization_small100.py

@@ -0,0 +1,365 @@
+# Copyright (c) 2022 Idiap Research Institute, http://www.idiap.ch/
+# Written by Alireza Mohammadshahi <alireza.mohammadshahi@idiap.ch>
+# This is a modified version of https://github.com/huggingface/transformers/blob/main/src/transformers/models/m2m_100/tokenization_m2m_100.py 
+# which owns by Fariseq Authors and The HuggingFace Inc. team.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for SMALL100."""
+import json
+import os
+from pathlib import Path
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import sentencepiece
+
+from transformers.tokenization_utils import BatchEncoding, PreTrainedTokenizer
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+SPIECE_UNDERLINE = "▁"
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "spm_file": "sentencepiece.bpe.model",
+    "tokenizer_config_file": "tokenizer_config.json",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "alirezamsh/small100": "https://huggingface.co/alirezamsh/small100/resolve/main/vocab.json",
+    },
+    "spm_file": {
+        "alirezamsh/small100": "https://huggingface.co/alirezamsh/small100/resolve/main/sentencepiece.bpe.model",
+    },
+    "tokenizer_config_file": {
+        "alirezamsh/small100": "https://huggingface.co/alirezamsh/small100/resolve/main/tokenizer_config.json",
+    },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "alirezamsh/small100": 1024,
+}
+
+# fmt: off
+FAIRSEQ_LANGUAGE_CODES = {
+    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"]
+}
+# fmt: on
+
+
+class SMALL100Tokenizer(PreTrainedTokenizer):
+    """
+    Construct an SMALL100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
+    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+    this superclass for more information regarding those methods.
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        spm_file (`str`):
+            Path to [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
+            contains the vocabulary.
+        tgt_lang (`str`, *optional*):
+            A string representing the target language.
+        eos_token (`str`, *optional*, defaults to `"</s>"`):
+            The end of sequence token.
+        sep_token (`str`, *optional*, defaults to `"</s>"`):
+            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
+            sequence classification or for a text and a question for question answering. It is also used as the last
+            token of a sequence built with special tokens.
+        unk_token (`str`, *optional*, defaults to `"<unk>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        pad_token (`str`, *optional*, defaults to `"<pad>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        language_codes (`str`, *optional*):
+            What language codes to use. Should be `"m2m100"`.
+        sp_model_kwargs (`dict`, *optional*):
+            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+            to set:
+            - `enable_sampling`: Enable subword regularization.
+            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
+              - `nbest_size = {0,1}`: No sampling is performed.
+              - `nbest_size > 1`: samples from the nbest_size results.
+              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
+                using forward-filtering-and-backward-sampling algorithm.
+            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
+              BPE-dropout.
+    Examples:
+    ```python
+    >>> from tokenization_small100 import SMALL100Tokenizer
+    >>> tokenizer = SMALL100Tokenizer.from_pretrained("alirezamsh/small100", tgt_lang="ro")
+    >>> src_text = " UN Chief Says There Is No Military Solution in Syria"
+    >>> tgt_text = "Şeful ONU declară că nu există o soluţie militară în Siria"
+    >>> model_inputs = tokenizer(src_text, text_target=tgt_text, return_tensors="pt")
+    >>> model(**model_inputs)  # should work
+    ```"""
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ["input_ids", "attention_mask"]
+
+    prefix_tokens: List[int] = []
+    suffix_tokens: List[int] = []
+
+    def __init__(
+        self,
+        vocab_file,
+        spm_file,
+        tgt_lang=None,
+        bos_token="<s>",
+        eos_token="</s>",
+        sep_token="</s>",
+        pad_token="<pad>",
+        unk_token="<unk>",
+        language_codes="m2m100",
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        num_madeup_words=8,
+        **kwargs,
+    ) -> None:
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+
+        self.language_codes = language_codes
+        fairseq_language_code = FAIRSEQ_LANGUAGE_CODES[language_codes]
+        self.lang_code_to_token = {lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code}
+
+        kwargs["additional_special_tokens"] = kwargs.get("additional_special_tokens", [])
+        kwargs["additional_special_tokens"] += [
+            self.get_lang_token(lang_code)
+            for lang_code in fairseq_language_code
+            if self.get_lang_token(lang_code) not in kwargs["additional_special_tokens"]
+        ]
+
+        self.vocab_file = vocab_file
+        self.encoder = load_json(vocab_file)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.spm_file = spm_file
+        self.sp_model = load_spm(spm_file, self.sp_model_kwargs)
+
+        self.encoder_size = len(self.encoder)
+
+        self.lang_token_to_id = {
+            self.get_lang_token(lang_code): self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code)
+        }
+        self.lang_code_to_id = {lang_code: self.encoder_size + i for i, lang_code in enumerate(fairseq_language_code)}
+        self.id_to_lang_token = {v: k for k, v in self.lang_token_to_id.items()}
+
+        self._tgt_lang = tgt_lang if tgt_lang is not None else "en"
+        self.cur_lang_id = self.get_lang_id(self._tgt_lang)
+        self.num_madeup_words = num_madeup_words
+        
+        super().__init__(
+            tgt_lang=tgt_lang,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            sep_token=sep_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            language_codes=language_codes,
+            sp_model_kwargs=self.sp_model_kwargs,
+            num_madeup_words=num_madeup_words,
+            **kwargs,
+        )
+        
+        self.set_lang_special_tokens(self._tgt_lang)
+
+
+    @property
+    def vocab_size(self) -> int:
+        return len(self.encoder) + len(self.lang_token_to_id) + self.num_madeup_words
+
+    @property
+    def tgt_lang(self) -> str:
+        return self._tgt_lang
+
+    @tgt_lang.setter
+    def tgt_lang(self, new_tgt_lang: str) -> None:
+        self._tgt_lang = new_tgt_lang
+        self.set_lang_special_tokens(self._tgt_lang)
+
+    def _tokenize(self, text: str) -> List[str]:
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        if token in self.lang_token_to_id:
+            return self.lang_token_to_id[token]
+        return self.encoder.get(token, self.encoder[self.unk_token])
+
+    def _convert_id_to_token(self, index: int) -> str:
+        """Converts an index (integer) in a token (str) using the decoder."""
+        if index in self.id_to_lang_token:
+            return self.id_to_lang_token[index]
+        return self.decoder.get(index, self.unk_token)
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        """Converts a sequence of tokens (strings for sub-words) in a single string."""
+        return self.sp_model.decode(tokens)
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        prefix_ones = [1] * len(self.prefix_tokens)
+        suffix_ones = [1] * len(self.suffix_tokens)
+        if token_ids_1 is None:
+            return prefix_ones + ([0] * len(token_ids_0)) + suffix_ones
+        return prefix_ones + ([0] * len(token_ids_0)) + ([0] * len(token_ids_1)) + suffix_ones
+
+    def build_inputs_with_special_tokens(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
+        adding special tokens. An MBART sequence has the following format, where `X` represents the sequence:
+        - `input_ids` (for encoder) `X [eos, src_lang_code]`
+        - `decoder_input_ids`: (for decoder) `X [eos, tgt_lang_code]`
+        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
+        separator.
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+        Returns:
+            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
+        """
+        if token_ids_1 is None:
+            if self.prefix_tokens is None:
+                return token_ids_0 + self.suffix_tokens
+            else:
+                return self.prefix_tokens + token_ids_0 + self.suffix_tokens
+        # We don't expect to process pairs, but leave the pair logic for API consistency
+        if self.prefix_tokens is None:
+            return token_ids_0 + token_ids_1 + self.suffix_tokens
+        else:
+            return self.prefix_tokens + token_ids_0 + token_ids_1 + self.suffix_tokens
+
+    def get_vocab(self) -> Dict:
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def __getstate__(self) -> Dict:
+        state = self.__dict__.copy()
+        state["sp_model"] = None
+        return state
+
+    def __setstate__(self, d: Dict) -> None:
+        self.__dict__ = d
+
+        # for backward compatibility
+        if not hasattr(self, "sp_model_kwargs"):
+            self.sp_model_kwargs = {}
+
+        self.sp_model = load_spm(self.spm_file, self.sp_model_kwargs)
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        save_dir = Path(save_directory)
+        if not save_dir.is_dir():
+            raise OSError(f"{save_directory} should be a directory")
+        vocab_save_path = save_dir / (
+            (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
+        )
+        spm_save_path = save_dir / (
+            (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
+        )
+
+        save_json(self.encoder, vocab_save_path)
+
+        if os.path.abspath(self.spm_file) != os.path.abspath(spm_save_path) and os.path.isfile(self.spm_file):
+            copyfile(self.spm_file, spm_save_path)
+        elif not os.path.isfile(self.spm_file):
+            with open(spm_save_path, "wb") as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (str(vocab_save_path), str(spm_save_path))
+
+    def prepare_seq2seq_batch(
+        self,
+        src_texts: List[str],
+        tgt_texts: Optional[List[str]] = None,
+        tgt_lang: str = "ro",
+        **kwargs,
+    ) -> BatchEncoding:
+        self.tgt_lang = tgt_lang
+        self.set_lang_special_tokens(self.tgt_lang)
+        return super().prepare_seq2seq_batch(src_texts, tgt_texts, **kwargs)
+
+    def _build_translation_inputs(self, raw_inputs, tgt_lang: Optional[str], **extra_kwargs):
+        """Used by translation pipeline, to prepare inputs for the generate function"""
+        if tgt_lang is None:
+            raise ValueError("Translation requires a `tgt_lang` for this model")
+        self.tgt_lang = tgt_lang
+        inputs = self(raw_inputs, add_special_tokens=True, **extra_kwargs)
+        return inputs
+
+    def _switch_to_input_mode(self):
+        self.set_lang_special_tokens(self.tgt_lang)
+
+    def _switch_to_target_mode(self):
+        self.prefix_tokens = None
+        self.suffix_tokens = [self.eos_token_id]        
+
+    def set_lang_special_tokens(self, src_lang: str) -> None:
+        """Reset the special tokens to the tgt lang setting. No prefix and suffix=[eos, tgt_lang_code]."""
+        lang_token = self.get_lang_token(src_lang)
+        self.cur_lang_id = self.lang_token_to_id[lang_token]
+        self.prefix_tokens = [self.cur_lang_id]
+        self.suffix_tokens = [self.eos_token_id]
+
+    def get_lang_token(self, lang: str) -> str:
+        return self.lang_code_to_token[lang]
+
+    def get_lang_id(self, lang: str) -> int:
+        lang_token = self.get_lang_token(lang)
+        return self.lang_token_to_id[lang_token]
+
+
+def load_spm(path: str, sp_model_kwargs: Dict[str, Any]) -> sentencepiece.SentencePieceProcessor:
+    spm = sentencepiece.SentencePieceProcessor(**sp_model_kwargs)
+    spm.Load(str(path))
+    return spm
+
+
+def load_json(path: str) -> Union[Dict, List]:
+    with open(path, "r") as f:
+        return json.load(f)
+
+
+def save_json(data, path: str) -> None:
+    with open(path, "w") as f:
+        json.dump(data, f, indent=2)

whisper_live/backend/translation_backend.py

@@ -0,0 +1,218 @@
+import json
+import logging
+import threading
+import time
+import queue
+from typing import Dict, Any, Optional
+import torch
+import threading
+from transformers import M2M100ForConditionalGeneration
+from whisper_live.backend.tokenization_small100 import SMALL100Tokenizer
+
+from whisper_live.backend.base import ServeClientBase
+
+
+class ServeClientTranslation(ServeClientBase):
+    """
+    Handles translation of completed transcription segments in a separate thread.
+    Reads from a queue populated by the transcription backend and sends translated
+    segments back to the client via WebSocket.
+    """
+    
+    def __init__(
+        self,
+        client_uid,
+        websocket,
+        translation_queue,
+        target_language="fr", 
+        send_last_n_segments=10,
+        model_name="alirezamsh/small100"
+    ):
+        """
+        Initialize the translation client.
+        
+        Args:
+            client_uid (str): Unique identifier for the client
+            websocket: WebSocket connection to the client
+            translation_queue (queue.Queue): Queue containing completed segments to translate
+            target_language (str): Target language code (default: "fr" for French)
+            send_last_n_segments (int): Number of recent translated segments to send
+            model_name (str): Translation model name to use
+        """
+        super().__init__(client_uid, websocket, send_last_n_segments)
+        self.translation_queue = translation_queue
+        self.target_language = target_language
+        self.model_name = model_name
+        self.translated_segments = []
+        self.translation_model = None
+        self.tokenizer = None
+        self.device = None
+        self.model_loaded = False
+        self.load_translation_model()
+        
+    def load_translation_model(self):
+        """Load the translation model and tokenizer."""
+        try:
+            self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            logging.info(f"Loading translation model on device: {self.device}")
+            
+            self.translation_model = M2M100ForConditionalGeneration.from_pretrained(
+                self.model_name
+            ).to(self.device)
+            self.tokenizer = SMALL100Tokenizer.from_pretrained(self.model_name)
+            self.tokenizer.tgt_lang = self.target_language
+            
+            self.model_loaded = True
+            logging.info(f"Translation model loaded successfully. Target language: {self.target_language}")
+        except Exception as e:
+            logging.error(f"Failed to load translation model: {e}")
+            self.translation_model = None
+            self.tokenizer = None
+            self.model_loaded = False
+    
+    def translate_text(self, text: str) -> str:
+        """
+        Translate a single text segment.
+        
+        Args:
+            text (str): Text to translate
+            
+        Returns:
+            str: Translated text or original text if translation fails
+        """
+        if not self.model_loaded or not text.strip():
+            return text
+            
+        try:
+            # Encode input and move to device
+            encoded_input = self.tokenizer(text, return_tensors="pt").to(self.device)
+            
+            # Generate translation
+            with torch.no_grad():
+                generated_tokens = self.translation_model.generate(**encoded_input)
+            
+            # Decode output
+            output = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+            return output[0] if output else text
+            
+        except Exception as e:
+            logging.error(f"Translation failed for text '{text}': {e}")
+            return text
+    
+    def process_translation_queue(self):
+        """
+        Process segments from the translation queue.
+        Continuously reads from the queue until None is received (exit signal).
+        """
+        logging.info(f"Starting translation processing for client {self.client_uid}")
+        
+        while not self.exit:
+            try:
+                # Get segment from queue with timeout
+                segment = self.translation_queue.get(timeout=1.0)
+                
+                # Check for exit signal
+                if segment is None:
+                    logging.info(f"Received exit signal for translation client {self.client_uid}")
+                    break
+                    
+                # Only translate completed segments
+                if not segment.get("completed", False):
+                    self.translation_queue.task_done()
+                    continue
+                    
+                # Translate the segment
+                original_text = segment.get("text", "")
+                translated_text = self.translate_text(original_text)
+                
+                # Create translated segment
+                translated_segment = {
+                    "start": segment["start"],
+                    "end": segment["end"],
+                    "text": translated_text,
+                    "completed": segment.get("completed", False),
+                    "target_language": self.target_language
+                }
+                
+                self.translated_segments.append(translated_segment)
+                segments_to_send = self.prepare_translated_segments()
+                self.send_translation_to_client(segments_to_send)
+                
+                self.translation_queue.task_done()
+                
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logging.error(f"Error processing translation queue: {e}")
+                continue
+        
+        logging.info(f"Translation processing ended for client {self.client_uid}")
+    
+    def prepare_translated_segments(self):
+        """
+        Prepare the last n translated segments to send to client.
+        
+        Returns:
+            list: List of recent translated segments
+        """
+        if len(self.translated_segments) >= self.send_last_n_segments:
+            return self.translated_segments[-self.send_last_n_segments:]
+        return self.translated_segments[:]
+    
+    def send_translation_to_client(self, translated_segments):
+        """
+        Send translated segments to the client via WebSocket.
+        
+        Args:
+            translated_segments (list): List of translated segments to send
+        """
+        try:
+            self.websocket.send(
+                json.dumps({
+                    "uid": self.client_uid,
+                    "translated_segments": translated_segments,
+                })
+            )
+        except Exception as e:
+            logging.error(f"[ERROR]: Sending translation data to client: {e}")
+    
+    def speech_to_text(self):
+        """
+        Override parent method to handle translation processing.
+        This method will be called when the translation thread starts.
+        """
+        self.process_translation_queue()
+    
+    def set_target_language(self, language: str):
+        """
+        Change the target language for translation.
+        
+        Args:
+            language (str): New target language code
+        """
+        self.target_language = language
+        if self.tokenizer:
+            self.tokenizer.tgt_lang = language
+            logging.info(f"Target language changed to: {language}")
+    
+    def cleanup(self):
+        """Clean up translation resources."""
+        logging.info(f"Cleaning up translation resources for client {self.client_uid}")
+        self.exit = True
+        
+        try:
+            self.translation_queue.put(None, timeout=1.0)
+        except:
+            pass
+        
+        self.translated_segments.clear()
+        
+        if self.translation_model:
+            del self.translation_model
+            self.translation_model = None
+        if self.tokenizer:
+            del self.tokenizer
+            self.tokenizer = None
+        
+        if self.device and self.device.type == 'cuda':
+            torch.cuda.empty_cache()

whisper_live/backend/trt_backend.py

@@ -0,0 +1,210 @@
+import json
+import logging
+import threading
+import time
+
+from whisper_live.backend.base import ServeClientBase
+from whisper_live.transcriber.transcriber_tensorrt import WhisperTRTLLM
+
+
+class ServeClientTensorRT(ServeClientBase):
+    SINGLE_MODEL = None
+    SINGLE_MODEL_LOCK = threading.Lock()
+
+    def __init__(
+        self,
+        websocket,
+        task="transcribe",
+        multilingual=False,
+        language=None,
+        client_uid=None,
+        model=None,
+        single_model=False,
+        use_py_session=False,
+        max_new_tokens=225,
+        send_last_n_segments=10,
+        no_speech_thresh=0.45,
+        clip_audio=False,
+        same_output_threshold=10,
+    ):
+        """
+        Initialize a ServeClient instance.
+        The Whisper model is initialized based on the client's language and device availability.
+        The transcription thread is started upon initialization. A "SERVER_READY" message is sent
+        to the client to indicate that the server is ready.
+
+        Args:
+            websocket (WebSocket): The WebSocket connection for the client.
+            task (str, optional): The task type, e.g., "transcribe." Defaults to "transcribe".
+            device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None.
+            multilingual (bool, optional): Whether the client supports multilingual transcription. Defaults to False.
+            language (str, optional): The language for transcription. Defaults to None.
+            client_uid (str, optional): A unique identifier for the client. Defaults to None.
+            single_model (bool, optional): Whether to instantiate a new model for each client connection. Defaults to False.
+            use_py_session (bool, optional): Use python session or cpp session. Defaults to Cpp Session.
+            max_new_tokens (int, optional): Max number of tokens to generate.
+            send_last_n_segments (int, optional): Number of most recent segments to send to the client. Defaults to 10.
+            no_speech_thresh (float, optional): Segments with no speech probability above this threshold will be discarded. Defaults to 0.45.
+            clip_audio (bool, optional): Whether to clip audio with no valid segments. Defaults to False.
+            same_output_threshold (int, optional): Number of repeated outputs before considering it as a valid segment. Defaults to 10.
+        """
+        super().__init__(
+            client_uid,
+            websocket,
+            send_last_n_segments,
+            no_speech_thresh,
+            clip_audio,
+            same_output_threshold,
+        )
+
+        self.language = language if multilingual else "en"
+        self.task = task
+        self.eos = False
+        self.max_new_tokens = max_new_tokens
+
+        if single_model:
+            if ServeClientTensorRT.SINGLE_MODEL is None:
+                self.create_model(model, multilingual, use_py_session=use_py_session)
+                ServeClientTensorRT.SINGLE_MODEL = self.transcriber
+            else:
+                self.transcriber = ServeClientTensorRT.SINGLE_MODEL
+        else:
+            self.create_model(model, multilingual, use_py_session=use_py_session)
+
+        # threading
+        self.trans_thread = threading.Thread(target=self.speech_to_text)
+        self.trans_thread.start()
+
+        self.websocket.send(json.dumps({
+            "uid": self.client_uid,
+            "message": self.SERVER_READY,
+            "backend": "tensorrt"
+        }))
+
+    def create_model(self, model, multilingual, warmup=True, use_py_session=False):
+        """
+        Instantiates a new model, sets it as the transcriber and does warmup if desired.
+        """
+        self.transcriber = WhisperTRTLLM(
+            model,
+            assets_dir="assets",
+            device="cuda",
+            is_multilingual=multilingual,
+            language=self.language,
+            task=self.task,
+            use_py_session=use_py_session,
+            max_output_len=self.max_new_tokens,
+        )
+        if warmup:
+            self.warmup()
+
+    def warmup(self, warmup_steps=10):
+        """
+        Warmup TensorRT since first few inferences are slow.
+
+        Args:
+            warmup_steps (int): Number of steps to warm up the model for.
+        """
+        logging.info("[INFO:] Warming up TensorRT engine..")
+        mel, _ = self.transcriber.log_mel_spectrogram("assets/jfk.flac")
+        for i in range(warmup_steps):
+            self.transcriber.transcribe(mel)
+
+    def set_eos(self, eos):
+        """
+        Sets the End of Speech (EOS) flag.
+
+        Args:
+            eos (bool): The value to set for the EOS flag.
+        """
+        self.lock.acquire()
+        self.eos = eos
+        self.lock.release()
+
+    def handle_transcription_output(self, last_segment, duration):
+        """
+        Handle the transcription output, updating the transcript and sending data to the client.
+
+        Args:
+            last_segment (str): The last segment from the whisper output which is considered to be incomplete because
+                                of the possibility of word being truncated.
+            duration (float): Duration of the transcribed audio chunk.
+        """
+        segments = self.prepare_segments({"text": last_segment})
+        self.send_transcription_to_client(segments)
+        if self.eos:
+            self.update_timestamp_offset(last_segment, duration)
+
+    def transcribe_audio(self, input_bytes):
+        """
+        Transcribe the audio chunk and send the results to the client.
+
+        Args:
+            input_bytes (np.array): The audio chunk to transcribe.
+        """
+        if ServeClientTensorRT.SINGLE_MODEL:
+            ServeClientTensorRT.SINGLE_MODEL_LOCK.acquire()
+        logging.info(f"[WhisperTensorRT:] Processing audio with duration: {input_bytes.shape[0] / self.RATE}")
+        mel, duration = self.transcriber.log_mel_spectrogram(input_bytes)
+        last_segment = self.transcriber.transcribe(
+            mel,
+            text_prefix=f"<|startoftranscript|><|{self.language}|><|{self.task}|><|notimestamps|>",
+        )
+        if ServeClientTensorRT.SINGLE_MODEL:
+            ServeClientTensorRT.SINGLE_MODEL_LOCK.release()
+        if last_segment:
+            self.handle_transcription_output(last_segment, duration)
+
+    def update_timestamp_offset(self, last_segment, duration):
+        """
+        Update timestamp offset and transcript.
+
+        Args:
+            last_segment (str): Last transcribed audio from the whisper model.
+            duration (float): Duration of the last audio chunk.
+        """
+        if not len(self.transcript):
+            self.transcript.append({"text": last_segment + " "})
+        elif self.transcript[-1]["text"].strip() != last_segment:
+            self.transcript.append({"text": last_segment + " "})
+        
+        with self.lock:
+            self.timestamp_offset += duration
+
+    def speech_to_text(self):
+        """
+        Process an audio stream in an infinite loop, continuously transcribing the speech.
+
+        This method continuously receives audio frames, performs real-time transcription, and sends
+        transcribed segments to the client via a WebSocket connection.
+
+        If the client's language is not detected, it waits for 30 seconds of audio input to make a language prediction.
+        It utilizes the Whisper ASR model to transcribe the audio, continuously processing and streaming results. Segments
+        are sent to the client in real-time, and a history of segments is maintained to provide context.
+
+        Raises:
+            Exception: If there is an issue with audio processing or WebSocket communication.
+
+        """
+        while True:
+            if self.exit:
+                logging.info("Exiting speech to text thread")
+                break
+
+            if self.frames_np is None:
+                time.sleep(0.02)    # wait for any audio to arrive
+                continue
+
+            self.clip_audio_if_no_valid_segment()
+
+            input_bytes, duration = self.get_audio_chunk_for_processing()
+            if duration < 0.4:
+                continue
+
+            try:
+                input_sample = input_bytes.copy()
+                logging.info(f"[WhisperTensorRT:] Processing audio with duration: {duration}")
+                self.transcribe_audio(input_sample)
+
+            except Exception as e:
+                logging.error(f"[ERROR]: {e}")

whisper_live/batch_inference.py

@@ -0,0 +1,397 @@
+"""
+Batch inference scheduler for WhisperLive.
+
+Replaces the per-session SINGLE_MODEL_LOCK with a queue-based batch system.
+Multiple sessions submit audio to a central queue; a single dedicated thread
+collects pending requests and runs them as a GPU batch via CTranslate2's
+batched encode() + generate() API.
+
+For batch_size=1, falls back to standard transcriber.transcribe() for
+identical behavior to the non-batched path.
+
+Usage:
+    Enable via ``--batch_inference`` CLI flag. The batch worker is lazily
+    started after the first client connects and the shared model is loaded.
+
+Thread safety:
+    - ``queue.Queue`` is stdlib thread-safe.
+    - Each ``BatchRequest.future`` (``threading.Event``) is written by the
+      batch worker BEFORE ``.set()``, read by the session thread AFTER
+      ``.wait()`` — no data race.
+    - Only the batch worker thread touches the GPU model — zero lock
+      contention between session threads.
+"""
+
+import logging
+import queue
+import threading
+import time
+from dataclasses import dataclass, field
+from math import ceil
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+from faster_whisper.audio import pad_or_trim
+from faster_whisper.tokenizer import Tokenizer
+from faster_whisper.vad import (
+    VadOptions,
+    collect_chunks,
+    get_speech_timestamps,
+)
+
+from whisper_live.transcriber.transcriber_faster_whisper import (
+    Segment,
+    TranscriptionInfo,
+    get_compression_ratio,
+    get_suppressed_tokens,
+)
+
+
+@dataclass
+class BatchRequest:
+    """A single inference request submitted by a session thread.
+
+    The session thread creates this, calls ``BatchInferenceWorker.submit()``,
+    then blocks on ``future.wait()``.  The batch worker fills ``result``
+    and/or ``error``, then signals ``future.set()``.
+
+    Attributes:
+        audio: Raw audio samples (float32, 16 kHz mono).
+        language: ISO language code or None for auto-detection.
+        task: ``"transcribe"`` or ``"translate"``.
+        initial_prompt: Optional prompt for Whisper conditioning.
+        use_vad: Whether to apply Voice Activity Detection.
+        vad_parameters: Parameters forwarded to ``VadOptions``.
+        future: Event signaled when the result is ready.
+        result: List of ``Segment`` objects (filled by worker).
+        info: ``TranscriptionInfo`` metadata (filled by worker).
+        error: Exception instance if processing failed.
+    """
+    audio: np.ndarray
+    language: Optional[str] = None
+    task: str = "transcribe"
+    initial_prompt: Optional[str] = None
+    use_vad: bool = True
+    vad_parameters: Optional[Dict] = None
+    # Signaling
+    future: threading.Event = field(default_factory=threading.Event)
+    # Results (filled by batch worker)
+    result: Optional[Any] = None
+    info: Optional[Any] = None
+    error: Optional[Exception] = None
+
+
+class BatchInferenceWorker:
+    """Central batch inference scheduler for the faster_whisper backend.
+
+    Owns a single daemon thread that is the **only** thread touching the GPU
+    model.  Per-session transcription threads submit ``BatchRequest`` objects
+    and block on ``future.wait()`` instead of competing for
+    ``SINGLE_MODEL_LOCK``.
+
+    The worker loop:
+
+    1. Blocks until the first request arrives from the queue.
+    2. Waits up to ``batch_window_ms`` for additional requests (up to
+       ``max_batch_size``).
+    3. Processes the collected batch:
+       - **batch_size == 1**: delegates to ``transcriber.transcribe()`` for
+         identical behavior to the non-batched path.
+       - **batch_size > 1**: runs a custom batched GPU path using
+         CTranslate2's ``encode()`` + ``generate()`` APIs.
+
+    Args:
+        transcriber: The shared ``WhisperModel`` instance.
+        max_batch_size: Maximum number of requests per batch.
+        batch_window_ms: Maximum time (ms) to wait for the batch to fill
+            after the first request arrives.
+    """
+
+    def __init__(
+        self,
+        transcriber,
+        max_batch_size: int = 8,
+        batch_window_ms: int = 50,
+    ):
+        self.transcriber = transcriber
+        self.max_batch_size = max_batch_size
+        self.batch_window_ms = batch_window_ms
+        self._queue: queue.Queue = queue.Queue()
+        self._stop_event = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+    def start(self):
+        """Start the background batch worker thread."""
+        self._thread = threading.Thread(target=self._worker_loop, daemon=True)
+        self._thread.start()
+        logging.info(
+            f"[BatchInference] Started (max_batch={self.max_batch_size}, "
+            f"window={self.batch_window_ms}ms)"
+        )
+
+    def stop(self):
+        """Signal the worker to stop and wait for it to finish."""
+        self._stop_event.set()
+        if self._thread:
+            self._thread.join(timeout=5)
+
+    def submit(self, request: BatchRequest):
+        """Submit an inference request to the batch queue.
+
+        Args:
+            request: The ``BatchRequest`` to enqueue.  The caller should
+                then call ``request.future.wait()`` to block until the
+                result is ready.
+        """
+        self._queue.put(request)
+
+    # -------------------------------------------------------------------------
+    # Worker loop
+    # -------------------------------------------------------------------------
+
+    def _worker_loop(self):
+        """Main loop: collect requests into batches and process them."""
+        while not self._stop_event.is_set():
+            batch: List[BatchRequest] = []
+
+            # Block until first request arrives
+            try:
+                first = self._queue.get(timeout=0.5)
+                batch.append(first)
+            except queue.Empty:
+                continue
+
+            # Collect more requests within the batch window
+            deadline = time.monotonic() + (self.batch_window_ms / 1000.0)
+            while len(batch) < self.max_batch_size:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    break
+                try:
+                    item = self._queue.get(timeout=remaining)
+                    batch.append(item)
+                except queue.Empty:
+                    break
+
+            # Process the collected batch
+            try:
+                self._process_batch(batch)
+            except Exception as e:
+                logging.error(f"[BatchInference] Batch processing error: {e}")
+                for req in batch:
+                    if not req.future.is_set():
+                        req.error = e
+                        req.future.set()
+
+    # -------------------------------------------------------------------------
+    # Batch processing
+    # -------------------------------------------------------------------------
+
+    def _process_batch(self, batch: List[BatchRequest]):
+        """Dispatch to single or multi-item processing."""
+        if len(batch) == 1:
+            self._process_single(batch[0])
+            return
+
+        logging.info(f"[BatchInference] Processing batch of {len(batch)}")
+        self._process_multi(batch)
+
+    def _process_single(self, req: BatchRequest):
+        """Process a single request using standard ``transcriber.transcribe()``.
+
+        This path is used when only one request is available in the batch
+        window, ensuring identical behavior to the non-batched code path.
+        """
+        try:
+            result, info = self.transcriber.transcribe(
+                req.audio,
+                language=req.language,
+                task=req.task,
+                initial_prompt=req.initial_prompt,
+                vad_filter=req.use_vad,
+                vad_parameters=req.vad_parameters if req.use_vad else None,
+            )
+            # Materialize the generator into a list
+            req.result = list(result) if result is not None else []
+            req.info = info
+        except Exception as e:
+            req.error = e
+        finally:
+            req.future.set()
+
+    def _process_multi(self, batch: List[BatchRequest]):
+        """Batched GPU path: encode + generate for multiple sessions at once.
+
+        Pipeline:
+            1. Per-item CPU preprocessing (VAD filtering + mel feature extraction)
+            2. Batch GPU encode — single ``transcriber.encode()`` call
+            3. Per-item prompt construction (handles different languages/tasks)
+            4. Batch GPU generate — single ``transcriber.model.generate()`` call
+            5. Per-item segment parsing and result dispatch
+        """
+        # Step 1: Per-item CPU preprocessing (VAD + feature extraction)
+        preprocessed = []
+        for req in batch:
+            try:
+                audio = req.audio
+                speech_chunks = None
+
+                if req.use_vad:
+                    vad_params = req.vad_parameters or {}
+                    vad_opts = VadOptions(**vad_params) if isinstance(vad_params, dict) else vad_params
+                    speech_chunks = get_speech_timestamps(audio, vad_opts)
+                    if speech_chunks:
+                        audio_chunks, _ = collect_chunks(audio, speech_chunks)
+                        audio = np.concatenate(audio_chunks, axis=0) if audio_chunks else audio
+
+                if audio.shape[0] == 0:
+                    # No speech detected — return empty result immediately
+                    req.result = []
+                    req.info = self._make_info(req, 0.0, 0.0)
+                    req.future.set()
+                    continue
+
+                duration = audio.shape[0] / self.transcriber.feature_extractor.sampling_rate
+                features = self.transcriber.feature_extractor(audio)
+                features = pad_or_trim(features)  # -> [n_mels, 3000]
+                preprocessed.append((req, features, audio, duration, speech_chunks))
+            except Exception as e:
+                req.error = e
+                req.future.set()
+
+        if not preprocessed:
+            return
+
+        try:
+            # Step 2: Batch GPU encode
+            feature_batch = np.stack([p[1] for p in preprocessed])  # [B, n_mels, 3000]
+            encoder_output = self.transcriber.encode(feature_batch)
+
+            # Step 3: Build per-item prompts (handles different languages/tasks)
+            tokenizers_list = []
+            prompts = []
+            resolved_languages = []
+
+            for i, (req, features, audio, duration, speech_chunks) in enumerate(preprocessed):
+                lang = req.language
+                # If language unknown, detect from encoder output
+                if lang is None:
+                    try:
+                        lang_results = self.transcriber.model.detect_language(encoder_output)
+                        if lang_results and len(lang_results) > i:
+                            detected = lang_results[i]
+                            if detected:
+                                lang = detected[0][0].strip("<|>")
+                    except Exception:
+                        lang = "en"  # fallback
+
+                resolved_languages.append(lang or "en")
+
+                tokenizer = Tokenizer(
+                    self.transcriber.hf_tokenizer,
+                    self.transcriber.model.is_multilingual,
+                    task=req.task,
+                    language=lang or "en",
+                )
+
+                previous_tokens = []
+                if req.initial_prompt:
+                    previous_tokens = tokenizer.encode(" " + req.initial_prompt.strip())
+
+                prompt = self.transcriber.get_prompt(
+                    tokenizer,
+                    previous_tokens=previous_tokens,
+                    without_timestamps=False,
+                )
+                tokenizers_list.append(tokenizer)
+                prompts.append(prompt)
+
+            # Step 4: Batch GPU generate
+            suppress_tokens = get_suppressed_tokens(tokenizers_list[0], [-1])
+
+            results = self.transcriber.model.generate(
+                encoder_output,
+                prompts,
+                beam_size=5,
+                patience=1,
+                length_penalty=1,
+                max_length=self.transcriber.max_length,
+                suppress_blank=True,
+                suppress_tokens=suppress_tokens,
+                return_scores=True,
+                return_no_speech_prob=True,
+                sampling_temperature=0.0,
+                repetition_penalty=1,
+                no_repeat_ngram_size=0,
+            )
+
+            # Step 5: Per-item segment parsing and result dispatch
+            for i, (req, features, audio, duration, speech_chunks) in enumerate(preprocessed):
+                try:
+                    tokenizer = tokenizers_list[i]
+                    gen_result = results[i]
+
+                    tokens = gen_result.sequences_ids[0]
+                    seq_len = len(tokens)
+                    cum_logprob = gen_result.scores[0] * seq_len
+                    avg_logprob = cum_logprob / (seq_len + 1) if seq_len > 0 else 0.0
+
+                    segment_size = int(ceil(duration) * self.transcriber.frames_per_second)
+
+                    subsegments, _, _ = self.transcriber._split_segments_by_timestamps(
+                        tokenizer=tokenizer,
+                        tokens=tokens,
+                        time_offset=0,
+                        segment_size=segment_size,
+                        segment_duration=duration,
+                        seek=0,
+                    )
+
+                    segments = []
+                    for seg_idx, subseg in enumerate(subsegments):
+                        text = tokenizer.decode(subseg["tokens"]).strip()
+                        if not text:
+                            continue
+                        segments.append(Segment(
+                            id=seg_idx,
+                            seek=subseg.get("seek", 0),
+                            start=subseg["start"],
+                            end=subseg["end"],
+                            text=text,
+                            tokens=subseg["tokens"],
+                            avg_logprob=avg_logprob,
+                            compression_ratio=get_compression_ratio(text),
+                            no_speech_prob=gen_result.no_speech_prob,
+                            words=None,
+                            temperature=0.0,
+                        ))
+
+                    req.result = segments
+                    req.info = self._make_info(
+                        req, duration, duration,
+                        language=resolved_languages[i],
+                    )
+                except Exception as e:
+                    req.error = e
+                finally:
+                    req.future.set()
+
+        except Exception as e:
+            logging.error(f"[BatchInference] GPU batch error: {e}")
+            for req, *_ in preprocessed:
+                if not req.future.is_set():
+                    req.error = e
+                    req.future.set()
+
+    def _make_info(self, req, duration, duration_after_vad, language=None):
+        """Build a ``TranscriptionInfo`` for the given request."""
+        return TranscriptionInfo(
+            language=language or req.language or "en",
+            language_probability=1.0,
+            duration=duration,
+            duration_after_vad=duration_after_vad,
+            all_language_probs=None,
+            transcription_options=None,
+            vad_options=None,
+        )

whisper_live/client.py

@@ -1,6 +1,8 @@
 import os
+import shutil
 import wave
 
+import logging
 import numpy as np
 import pyaudio
 import threading
@@ -8,13 +10,13 @@ import json
 import websocket
 import uuid
 import time
-import ffmpeg
+import av
 import whisper_live.utils as utils
 
 
 class Client:
     """
-    Handles audio recording, streaming, and communication with a server using WebSocket.
+    Handles communication with a server using WebSocket.
     """
     INSTANCES = {}
     END_OF_AUDIO = "END_OF_AUDIO"
@@ -27,7 +29,20 @@ class Client:
         translate=False,
         model="small",
         srt_file_path="output.srt",
-        use_vad=True
+        use_vad=True,
+        use_wss=False,
+        log_transcription=True,
+        send_last_n_segments=10,
+        no_speech_thresh=0.45,
+        clip_audio=False,
+        same_output_threshold=10,
+        transcription_callback=None,
+        enable_translation=False,
+        target_language="fr",
+        translation_callback=None,
+        translation_srt_file_path="output_translated.srt",
+        enable_timestamps=False,
+        display_segments=4,
     ):
         """
         Initializes a Client instance for audio recording and streaming to a server.
@@ -41,41 +56,57 @@ class Client:
             port (int): The port number for the WebSocket server.
             lang (str, optional): The selected language for transcription. Default is None.
             translate (bool, optional): Specifies if the task is translation. Default is False.
+            model (str, optional): The whisper model to use (e.g., "small", "medium", "large"). Default is "small".
+            srt_file_path (str, optional): The file path to save the output SRT file. Default is "output.srt".
+            use_vad (bool, optional): Whether to enable voice activity detection. Default is True.
+            log_transcription (bool, optional): Whether to log transcription output to the console. Default is True.
+            send_last_n_segments (int, optional): Number of most recent segments to send to the client. Defaults to 10.
+            no_speech_thresh (float, optional): Segments with no speech probability above this threshold will be discarded. Defaults to 0.45.
+            clip_audio (bool, optional): Whether to clip audio with no valid segments. Defaults to False.
+            same_output_threshold (int, optional): Number of repeated outputs before considering it as a valid segment. Defaults to 10.
+            transcription_callback (callable, optional): A callback function to handle transcription results. Default is None.
+            enable_translation (float, optional): Whether to enable translation from any to any language. Defaults to False.
+            target_language (str, optional): Target language for translation. Defaults to 'fr'.
+            translation_callback (callable, optional): A callback function to handle translation results. Default is None.
+            translation_srt_file_path (str, optional): The file path to save the translated output SRT file. Default is "output_translated.srt".
         """
-        self.chunk = 4096
-        self.format = pyaudio.paInt16
-        self.channels = 1
-        self.rate = 16000
-        self.record_seconds = 60000
         self.recording = False
         self.task = "transcribe"
         self.uid = str(uuid.uuid4())
         self.waiting = False
-        self.last_response_recieved = None
+        self.last_response_received = None
         self.disconnect_if_no_response_for = 15
         self.language = lang
         self.model = model
         self.server_error = False
         self.srt_file_path = srt_file_path
         self.use_vad = use_vad
-        self.last_recieved_segment = None
+        self.use_wss = use_wss
+        self.last_segment = None
+        self.last_received_segment = None
+        self.log_transcription = log_transcription
+        self.send_last_n_segments = send_last_n_segments
+        self.no_speech_thresh = no_speech_thresh
+        self.clip_audio = clip_audio
+        self.same_output_threshold = same_output_threshold
+        self.transcription_callback = transcription_callback
 
+        # Translation-specific attributes
+        self.enable_translation = enable_translation
+        self.target_language = target_language
+        self.translation_callback = translation_callback
+        self.translation_srt_file_path = translation_srt_file_path
+        self.last_translated_segment = None
         if translate:
             self.task = "translate"
+        self.enable_timestamps = enable_timestamps
+        self.display_segments = display_segments
 
-        self.timestamp_offset = 0.0
         self.audio_bytes = None
-        self.p = pyaudio.PyAudio()
-        self.stream = self.p.open(
-            format=self.format,
-            channels=self.channels,
-            rate=self.rate,
-            input=True,
-            frames_per_buffer=self.chunk,
-        )
 
         if host is not None and port is not None:
-            socket_url = f"ws://{host}:{port}"
+            socket_protocol = 'wss' if self.use_wss else "ws"
+            socket_url = f"{socket_protocol}://{host}:{port}"
             self.client_socket = websocket.WebSocketApp(
                 socket_url,
                 on_open=lambda ws: self.on_open(ws),
@@ -93,11 +124,11 @@ class Client:
 
         # start websocket client in a thread
         self.ws_thread = threading.Thread(target=self.client_socket.run_forever)
-        self.ws_thread.setDaemon(True)
+        self.ws_thread.daemon = True
         self.ws_thread.start()
 
-        self.frames = b""
         self.transcript = []
+        self.translated_transcript = []
         print("[INFO]: * recording")
 
     def handle_status_messages(self, message_data):
@@ -112,27 +143,77 @@ class Client:
         elif status == "WARNING":
             print(f"Message from Server: {message_data['message']}")
 
-    def process_segments(self, segments):
+    def process_segments(self, segments, translated=False):
         """Processes transcript segments."""
         text = []
         for i, seg in enumerate(segments):
             if not text or text[-1] != seg["text"]:
-                text.append(seg["text"])
-                if i == len(segments) - 1:
+                text.append(seg["text"].strip())
+                if i == len(segments) - 1 and not seg.get("completed", False):
                     self.last_segment = seg
-                elif (self.server_backend == "faster_whisper" and
-                      (not self.transcript or
-                        float(seg['start']) >= float(self.transcript[-1]['end']))):
-                    self.transcript.append(seg)
-        # update last received segment and last valild responsne time
-        if self.last_recieved_segment is None or self.last_recieved_segment != segments[-1]["text"]:
-            self.last_response_recieved = time.time()
-            self.last_recieved_segment = segments[-1]["text"]
+                elif self.server_backend == "faster_whisper" and seg.get("completed", False):
+                    if translated:
+                        if (not self.translated_transcript or float(seg['start']) >= float(self.translated_transcript[-1]['end'])):
+                            self.translated_transcript.append(seg)
+                    else:
+                        if (not self.transcript or float(seg['start']) >= float(self.transcript[-1]['end'])):
+                            self.transcript.append(seg)
+        # update last received segment and last valid response time
+        if not translated:
+            if self.last_received_segment is None or self.last_received_segment != segments[-1]["text"]:
+                self.last_response_received = time.time()
+                self.last_received_segment = segments[-1]["text"]
+
+        # call the transcription callback if provided
+        if translated:
+            if self.translation_callback and callable(self.translation_callback):
+                try:
+                    self.translation_callback(" ".join(text), segments) # string, list
+                except Exception as e:
+                    print(f"[WARN] translation_callback raised: {e}")
+                return
+        else:
+            if self.transcription_callback and callable(self.transcription_callback):
+                try:
+                    self.transcription_callback(" ".join(text), segments) # string, list
+                except Exception as e:
+                    print(f"[WARN] transcription_callback raised: {e}")
+                return
+
+        if self.log_transcription:
+            if self.enable_timestamps:
+                original_text_with_timestamps = [
+                    {"start": seg["start"], "end": seg["end"], "text": seg["text"]}
+                    for seg in self.transcript[-self.display_segments:]]
+                if self.last_segment is not None and not any(
+                    data.get("text") == self.last_segment["text"]
+                    for data in original_text_with_timestamps):
+                    original_text_with_timestamps.append({
+                        "start": self.last_segment["start"],
+                        "end": self.last_segment["end"],
+                        "text": self.last_segment["text"]
+                    })
+                utils.clear_screen()
+                utils.print_transcript(original_text_with_timestamps, timestamps=True)
+
+                if self.enable_translation:
+                    print(f"\n\nTRANSLATION to {self.target_language}:")
+                    utils.print_transcript([
+                        {"start": seg["start"], "end": seg["end"], "text": seg["text"]}
+                        for seg in self.translated_transcript[-self.display_segments:]
+                    ], timestamps=True)
+
+            else:
+                original_text = [seg["text"] for seg in self.transcript[-self.display_segments:]]
+                if self.last_segment is not None and self.last_segment["text"] not in original_text:
+                    original_text.append(self.last_segment["text"])
+                utils.clear_screen()
+                utils.print_transcript(original_text)
+
+                if self.enable_translation:
+                    print(f"\n\nTRANSLATION to {self.target_language}:")
+                    utils.print_transcript([seg["text"] for seg in self.translated_transcript[-self.display_segments:]], translated=True)
 
-        # Truncate to last 3 entries for brevity.
-        text = text[-3:]
-        utils.clear_screen()
-        utils.print_transcript(text)
 
     def on_message(self, ws, message):
         """
@@ -162,7 +243,7 @@ class Client:
             self.recording = False
 
         if "message" in message.keys() and message["message"] == "SERVER_READY":
-            self.last_response_recieved = time.time()
+            self.last_response_received = time.time()
             self.recording = True
             self.server_backend = message["backend"]
             print(f"[INFO]: Server Running with backend {self.server_backend}")
@@ -178,6 +259,9 @@ class Client:
 
         if "segments" in message.keys():
             self.process_segments(message["segments"])
+        
+        if "translated_segments" in message.keys():
+            self.process_segments(message["translated_segments"], translated=True)
 
     def on_error(self, ws, error):
         print(f"[ERROR] WebSocket Error: {error}")
@@ -187,7 +271,6 @@ class Client:
     def on_close(self, ws, close_status_code, close_msg):
         print(f"[INFO]: Websocket connection closed: {close_status_code}: {close_msg}")
         self.recording = False
-        self.server_error = False
         self.waiting = False
 
     def on_open(self, ws):
@@ -209,28 +292,17 @@ class Client:
                     "language": self.language,
                     "task": self.task,
                     "model": self.model,
-                    "use_vad": self.use_vad
+                    "use_vad": self.use_vad,
+                    "send_last_n_segments": self.send_last_n_segments,
+                    "no_speech_thresh": self.no_speech_thresh,
+                    "clip_audio": self.clip_audio,
+                    "same_output_threshold": self.same_output_threshold,
+                    "enable_translation": self.enable_translation,
+                    "target_language": self.target_language,
                 }
             )
         )
 
-    @staticmethod
-    def bytes_to_float_array(audio_bytes):
-        """
-        Convert audio data from bytes to a NumPy float array.
-
-        It assumes that the audio data is in 16-bit PCM format. The audio data is normalized to
-        have values between -1 and 1.
-
-        Args:
-            audio_bytes (bytes): Audio data in bytes.
-
-        Returns:
-            np.ndarray: A NumPy array containing the audio data as float values normalized between -1 and 1.
-        """
-        raw_data = np.frombuffer(buffer=audio_bytes, dtype=np.int16)
-        return raw_data.astype(np.float32) / 32768.0
-
     def send_packet_to_server(self, message):
         """
         Send an audio packet to the server using WebSocket.
@@ -244,62 +316,6 @@ class Client:
         except Exception as e:
             print(e)
 
-    def play_file(self, filename):
-        """
-        Play an audio file and send it to the server for processing.
-
-        Reads an audio file, plays it through the audio output, and simultaneously sends
-        the audio data to the server for processing. It uses PyAudio to create an audio
-        stream for playback. The audio data is read from the file in chunks, converted to
-        floating-point format, and sent to the server using WebSocket communication.
-        This method is typically used when you want to process pre-recorded audio and send it
-        to the server in real-time.
-
-        Args:
-            filename (str): The path to the audio file to be played and sent to the server.
-        """
-
-        # read audio and create pyaudio stream
-        with wave.open(filename, "rb") as wavfile:
-            self.stream = self.p.open(
-                format=self.p.get_format_from_width(wavfile.getsampwidth()),
-                channels=wavfile.getnchannels(),
-                rate=wavfile.getframerate(),
-                input=True,
-                output=True,
-                frames_per_buffer=self.chunk,
-            )
-            try:
-                while self.recording:
-                    data = wavfile.readframes(self.chunk)
-                    if data == b"":
-                        break
-
-                    audio_array = self.bytes_to_float_array(data)
-                    self.send_packet_to_server(audio_array.tobytes())
-                    self.stream.write(data)
-
-                wavfile.close()
-
-                assert self.last_response_recieved
-                while time.time() - self.last_response_recieved < self.disconnect_if_no_response_for:
-                    continue
-                self.send_packet_to_server(Client.END_OF_AUDIO.encode('utf-8'))
-                if self.server_backend == "faster_whisper":
-                    self.write_srt_file(self.srt_file_path)
-                self.stream.close()
-                self.close_websocket()
-
-            except KeyboardInterrupt:
-                wavfile.close()
-                self.stream.stop_stream()
-                self.stream.close()
-                self.p.terminate()
-                self.close_websocket()
-                if self.server_backend == "faster_whisper":
-                    self.write_srt_file(self.srt_file_path)
-                print("[INFO]: Keyboard interrupt.")
-
     def close_websocket(self):
         """
         Close the WebSocket connection and join the WebSocket thread.
@@ -327,6 +343,344 @@ class Client:
         """
         return self.client_socket
 
+    def write_srt_file(self, output_path="output.srt"):
+        """
+        Writes out the transcript in .srt format.
+
+        Args:
+            message (output_path, optional): The path to the target file.  Default is "output.srt".
+
+        """
+        if self.server_backend == "faster_whisper":
+            if not self.transcript and self.last_segment is not None:
+                self.transcript.append(self.last_segment)
+            elif self.last_segment and self.transcript[-1]["text"] != self.last_segment["text"]:
+                self.transcript.append(self.last_segment)
+            utils.create_srt_file(self.transcript, output_path)
+
+        if self.enable_translation:
+            utils.create_srt_file(self.translated_transcript, self.translation_srt_file_path)
+
+    def wait_before_disconnect(self):
+        """Waits a bit before disconnecting in order to process pending responses."""
+        assert self.last_response_received
+        while time.time() - self.last_response_received < self.disconnect_if_no_response_for:
+            continue
+
+
+class TranscriptionTeeClient:
+    """
+    Client for handling audio recording, streaming, and transcription tasks via one or more
+    WebSocket connections.
+
+    Acts as a high-level client for audio transcription tasks using a WebSocket connection. It can be used
+    to send audio data for transcription to one or more servers, and receive transcribed text segments.
+    Args:
+        clients (list): one or more previously initialized Client instances
+
+    Attributes:
+        clients (list): the underlying Client instances responsible for handling WebSocket connections.
+    """
+    def __init__(self, clients, save_output_recording=False, output_recording_filename="./output_recording.wav", mute_audio_playback=False):
+        self.clients = clients
+        if not self.clients:
+            raise Exception("At least one client is required.")
+        self.chunk = 4096
+        self.format = pyaudio.paInt16
+        self.channels = 1
+        self.rate = 16000
+        self.record_seconds = 60000
+        self.save_output_recording = save_output_recording
+        self.output_recording_filename = output_recording_filename
+        self.mute_audio_playback = mute_audio_playback
+        self.frames = b""
+        self.p = pyaudio.PyAudio()
+        try:
+            self.stream = self.p.open(
+                format=self.format,
+                channels=self.channels,
+                rate=self.rate,
+                input=True,
+                frames_per_buffer=self.chunk,
+            )
+        except OSError as error:
+            print(f"[WARN]: Unable to access microphone. {error}")
+            self.stream = None
+
+    def __call__(self, audio=None, rtsp_url=None, hls_url=None, save_file=None):
+        """
+        Start the transcription process.
+
+        Initiates the transcription process by connecting to the server via a WebSocket. It waits for the server
+        to be ready to receive audio data and then sends audio for transcription. If an audio file is provided, it
+        will be played and streamed to the server; otherwise, it will perform live recording.
+
+        Args:
+            audio (str, optional): Path to an audio file for transcription. Default is None, which triggers live recording.
+
+        """
+        assert sum(
+            source is not None for source in [audio, rtsp_url, hls_url]
+        ) <= 1, 'You must provide only one selected source'
+
+        print("[INFO]: Waiting for server ready ...")
+        for client in self.clients:
+            while not client.recording:
+                if client.waiting or client.server_error:
+                    self.close_all_clients()
+                    return
+
+        print("[INFO]: Server Ready!")
+        if hls_url is not None:
+            self.process_hls_stream(hls_url, save_file)
+        elif audio is not None:
+            resampled_file = utils.resample(audio)
+            self.play_file(resampled_file)
+        elif rtsp_url is not None:
+            self.process_rtsp_stream(rtsp_url)
+        else:
+            self.record()
+
+    def close_all_clients(self):
+        """Closes all client websockets."""
+        for client in self.clients:
+            client.close_websocket()
+
+    def write_all_clients_srt(self):
+        """Writes out .srt files for all clients."""
+        for client in self.clients:
+            client.write_srt_file(client.srt_file_path)
+
+    def multicast_packet(self, packet, unconditional=False):
+        """
+        Sends an identical packet via all clients.
+
+        Args:
+            packet (bytes): The audio data packet in bytes to be sent.
+            unconditional (bool, optional): If true, send regardless of whether clients are recording.  Default is False.
+        """
+        for client in self.clients:
+            if (unconditional or client.recording):
+                client.send_packet_to_server(packet)
+
+    def play_file(self, filename):
+        """
+        Play an audio file and send it to the server for processing.
+
+        Reads an audio file, plays it through the audio output, and simultaneously sends
+        the audio data to the server for processing. It uses PyAudio to create an audio
+        stream for playback. The audio data is read from the file in chunks, converted to
+        floating-point format, and sent to the server using WebSocket communication.
+        This method is typically used when you want to process pre-recorded audio and send it
+        to the server in real-time.
+
+        Args:
+            filename (str): The path to the audio file to be played and sent to the server.
+        """
+
+        # read audio and create pyaudio stream
+        with wave.open(filename, "rb") as wavfile:
+            if self.mute_audio_playback:
+                self.stream = None
+            else:
+                self.stream = self.p.open(
+                    format=self.p.get_format_from_width(wavfile.getsampwidth()),
+                    channels=wavfile.getnchannels(),
+                    rate=wavfile.getframerate(),
+                    input=True,
+                    output=True,
+                    frames_per_buffer=self.chunk,
+                )
+
+            chunk_duration = self.chunk / float(wavfile.getframerate())
+            try:
+                while any(client.recording for client in self.clients):
+                    data = wavfile.readframes(self.chunk)
+                    if data == b"":
+                        break
+
+                    audio_array = self.bytes_to_float_array(data)
+                    self.multicast_packet(audio_array.tobytes())
+                    if self.mute_audio_playback:
+                        time.sleep(chunk_duration)
+                    else:
+                        self.stream.write(data)
+    
+                wavfile.close()
+
+                for client in self.clients:
+                    client.wait_before_disconnect()
+                self.multicast_packet(Client.END_OF_AUDIO.encode('utf-8'), True)
+                self.write_all_clients_srt()
+                if self.stream:
+                    self.stream.close()
+                self.close_all_clients()
+
+            except KeyboardInterrupt:
+                wavfile.close()
+                self.stream.stop_stream()
+                self.stream.close()
+                self.p.terminate()
+                self.close_all_clients()
+                self.write_all_clients_srt()
+                print("[INFO]: Keyboard interrupt.")
+
+    def process_rtsp_stream(self, rtsp_url):
+        """
+        Connect to an RTSP source, process the audio stream, and send it for transcription.
+
+        Args:
+            rtsp_url (str): The URL of the RTSP stream source.
+        """
+        print("[INFO]: Connecting to RTSP stream...")
+        try:
+            container = av.open(rtsp_url, format="rtsp", options={"rtsp_transport": "tcp"})
+            self.process_av_stream(container, stream_type="RTSP")
+        except Exception as e:
+            print(f"[ERROR]: Failed to process RTSP stream: {e}")
+        finally:
+            for client in self.clients:
+                client.wait_before_disconnect()
+            self.multicast_packet(Client.END_OF_AUDIO.encode('utf-8'), True)
+            self.close_all_clients()
+            self.write_all_clients_srt()
+        print("[INFO]: RTSP stream processing finished.")
+
+    def process_hls_stream(self, hls_url, save_file=None):
+        """
+        Connect to an HLS source, process the audio stream, and send it for transcription.
+
+        Args:
+            hls_url (str): The URL of the HLS stream source.
+            save_file (str, optional): Local path to save the network stream.
+        """
+        print("[INFO]: Connecting to HLS stream...")
+        try:
+            container = av.open(hls_url, format="hls")
+            self.process_av_stream(container, stream_type="HLS", save_file=save_file)
+        except Exception as e:
+            print(f"[ERROR]: Failed to process HLS stream: {e}")
+        finally:
+            for client in self.clients:
+                client.wait_before_disconnect()
+            self.multicast_packet(Client.END_OF_AUDIO.encode('utf-8'), True)
+            self.close_all_clients()
+            self.write_all_clients_srt()
+        print("[INFO]: HLS stream processing finished.")
+
+    def process_av_stream(self, container, stream_type, save_file=None):
+        """
+        Process an AV container stream and send audio packets to the server.
+
+        Args:
+            container (av.container.InputContainer): The input container to process.
+            stream_type (str): The type of stream being processed ("RTSP" or "HLS").
+            save_file (str, optional): Local path to save the stream. Default is None.
+        """
+        audio_stream = next((s for s in container.streams if s.type == "audio"), None)
+        if not audio_stream:
+            print(f"[ERROR]: No audio stream found in {stream_type} source.")
+            return
+
+        output_container = None
+        if save_file:
+            output_container = av.open(save_file, mode="w")
+            output_audio_stream = output_container.add_stream(codec_name="pcm_s16le", rate=self.rate)
+
+        try:
+            for packet in container.demux(audio_stream):
+                for frame in packet.decode():
+                    audio_data = frame.to_ndarray().tobytes()
+                    self.multicast_packet(audio_data)
+
+                    if save_file:
+                        output_container.mux(frame)
+        except Exception as e:
+            print(f"[ERROR]: Error during {stream_type} stream processing: {e}")
+        finally:
+            # Wait for server to send any leftover transcription.
+            time.sleep(5)
+            self.multicast_packet(Client.END_OF_AUDIO.encode('utf-8'), True)
+            if output_container:
+                output_container.close()
+            container.close()
+
+    def save_chunk(self, n_audio_file):
+        """
+        Saves the current audio frames to a WAV file in a separate thread.
+
+        Args:
+        n_audio_file (int): The index of the audio file which determines the filename.
+                            This helps in maintaining the order and uniqueness of each chunk.
+        """
+        t = threading.Thread(
+            target=self.write_audio_frames_to_file,
+            args=(self.frames[:], f"chunks/{n_audio_file}.wav",),
+        )
+        t.start()
+
+    def finalize_recording(self, n_audio_file):
+        """
+        Finalizes the recording process by saving any remaining audio frames,
+        closing the audio stream, and terminating the process.
+
+        Args:
+        n_audio_file (int): The file index to be used if there are remaining audio frames to be saved.
+                            This index is incremented before use if the last chunk is saved.
+        """
+        if self.save_output_recording and len(self.frames):
+            self.write_audio_frames_to_file(
+                self.frames[:], f"chunks/{n_audio_file}.wav"
+            )
+            n_audio_file += 1
+        self.stream.stop_stream()
+        self.stream.close()
+        self.p.terminate()
+        self.close_all_clients()
+        if self.save_output_recording:
+            self.write_output_recording(n_audio_file)
+        self.write_all_clients_srt()
+
+    def record(self):
+        """
+        Record audio data from the input stream and save it to a WAV file.
+
+        Continuously records audio data from the input stream, sends it to the server via a WebSocket
+        connection, and simultaneously saves it to multiple WAV files in chunks. It stops recording when
+        the `RECORD_SECONDS` duration is reached or when the `RECORDING` flag is set to `False`.
+
+        Audio data is saved in chunks to the "chunks" directory. Each chunk is saved as a separate WAV file.
+        The recording will continue until the specified duration is reached or until the `RECORDING` flag is set to `False`.
+        The recording process can be interrupted by sending a KeyboardInterrupt (e.g., pressing Ctrl+C). After recording,
+        the method combines all the saved audio chunks into the specified `out_file`.
+        """
+        n_audio_file = 0
+        if self.save_output_recording:
+            if os.path.exists("chunks"):
+                shutil.rmtree("chunks")
+            os.makedirs("chunks")
+        try:
+            for _ in range(0, int(self.rate / self.chunk * self.record_seconds)):
+                if not any(client.recording for client in self.clients):
+                    break
+                data = self.stream.read(self.chunk, exception_on_overflow=False)
+                self.frames += data
+
+                audio_array = self.bytes_to_float_array(data)
+
+                self.multicast_packet(audio_array.tobytes())
+
+                # save frames if more than a minute
+                if len(self.frames) > 60 * self.rate:
+                    if self.save_output_recording:
+                        self.save_chunk(n_audio_file)
+                        n_audio_file += 1
+                    self.frames = b""
+            self.write_all_clients_srt()
+
+        except KeyboardInterrupt:
+            self.finalize_recording(n_audio_file)
+
     def write_audio_frames_to_file(self, frames, file_name):
         """
         Write audio frames to a WAV file.
@@ -346,104 +700,7 @@ class Client:
             wavfile.setframerate(self.rate)
             wavfile.writeframes(frames)
 
-    def process_hls_stream(self, hls_url):
-        """
-        Connect to an HLS source, process the audio stream, and send it for transcription.
-
-        Args:
-            hls_url (str): The URL of the HLS stream source.
-        """
-        print("[INFO]: Connecting to HLS stream...")
-        process = None  # Initialize process to None
-
-        try:
-            # Connecting to the HLS stream using ffmpeg-python
-            process = (
-                ffmpeg
-                .input(hls_url, threads=0)
-                .output('-', format='s16le', acodec='pcm_s16le', ac=1, ar=self.rate)
-                .run_async(pipe_stdout=True, pipe_stderr=True)
-            )
-
-            # Process the stream
-            while True:
-                in_bytes = process.stdout.read(self.chunk * 2)  # 2 bytes per sample
-                if not in_bytes:
-                    break
-                audio_array = self.bytes_to_float_array(in_bytes)
-                self.send_packet_to_server(audio_array.tobytes())
-
-        except Exception as e:
-            print(f"[ERROR]: Failed to connect to HLS stream: {e}")
-        finally:
-            if process:
-                process.kill()
-
-        print("[INFO]: HLS stream processing finished.")
-
-    def record(self, out_file="output_recording.wav"):
-        """
-        Record audio data from the input stream and save it to a WAV file.
-
-        Continuously records audio data from the input stream, sends it to the server via a WebSocket
-        connection, and simultaneously saves it to multiple WAV files in chunks. It stops recording when
-        the `RECORD_SECONDS` duration is reached or when the `RECORDING` flag is set to `False`.
-
-        Audio data is saved in chunks to the "chunks" directory. Each chunk is saved as a separate WAV file.
-        The recording will continue until the specified duration is reached or until the `RECORDING` flag is set to `False`.
-        The recording process can be interrupted by sending a KeyboardInterrupt (e.g., pressing Ctrl+C). After recording,
-        the method combines all the saved audio chunks into the specified `out_file`.
-
-        Args:
-            out_file (str, optional): The name of the output WAV file to save the entire recording.
-                                      Default is "output_recording.wav".
-
-        """
-        n_audio_file = 0
-        if not os.path.exists("chunks"):
-            os.makedirs("chunks", exist_ok=True)
-        try:
-            for _ in range(0, int(self.rate / self.chunk * self.record_seconds)):
-                if not self.recording:
-                    break
-                data = self.stream.read(self.chunk, exception_on_overflow=False)
-                self.frames += data
-
-                audio_array = Client.bytes_to_float_array(data)
-
-                self.send_packet_to_server(audio_array.tobytes())
-
-                # save frames if more than a minute
-                if len(self.frames) > 60 * self.rate:
-                    t = threading.Thread(
-                        target=self.write_audio_frames_to_file,
-                        args=(
-                            self.frames[:],
-                            f"chunks/{n_audio_file}.wav",
-                        ),
-                    )
-                    t.start()
-                    n_audio_file += 1
-                    self.frames = b""
-            if self.server_backend == "faster_whisper":
-                self.write_srt_file(self.srt_file_path)
-
-        except KeyboardInterrupt:
-            if len(self.frames):
-                self.write_audio_frames_to_file(
-                    self.frames[:], f"chunks/{n_audio_file}.wav"
-                )
-                n_audio_file += 1
-            self.stream.stop_stream()
-            self.stream.close()
-            self.p.terminate()
-            self.close_websocket()
-
-            self.write_output_recording(n_audio_file, out_file)
-            if self.server_backend == "faster_whisper":
-                self.write_srt_file(self.srt_file_path)
-
-    def write_output_recording(self, n_audio_file, out_file):
+    def write_output_recording(self, n_audio_file):
         """
         Combine and save recorded audio chunks into a single WAV file.
 
@@ -462,7 +719,7 @@ class Client:
             for i in range(n_audio_file)
             if os.path.exists(f"chunks/{i}.wav")
         ]
-        with wave.open(out_file, "wb") as wavfile:
+        with wave.open(self.output_recording_filename, "wb") as wavfile:
             wavfile: wave.Wave_write
             wavfile.setnchannels(self.channels)
             wavfile.setsampwidth(2)
@@ -477,24 +734,56 @@ class Client:
                 # remove this file
                 os.remove(in_file)
         wavfile.close()
+        # clean up temporary directory to store chunks
+        if os.path.exists("chunks"):
+            shutil.rmtree("chunks")
 
-    def write_srt_file(self, output_path="output.srt"):
-        self.transcript.append(self.last_segment)
-        utils.create_srt_file(self.transcript, output_path)
+    @staticmethod
+    def bytes_to_float_array(audio_bytes):
+        """
+        Convert audio data from bytes to a NumPy float array.
+
+        It assumes that the audio data is in 16-bit PCM format. The audio data is normalized to
+        have values between -1 and 1.
+
+        Args:
+            audio_bytes (bytes): Audio data in bytes.
+
+        Returns:
+            np.ndarray: A NumPy array containing the audio data as float values normalized between -1 and 1.
+        """
+        raw_data = np.frombuffer(buffer=audio_bytes, dtype=np.int16)
+        return raw_data.astype(np.float32) / 32768.0
 
 
-class TranscriptionClient:
+class TranscriptionClient(TranscriptionTeeClient):
     """
-    Client for handling audio transcription tasks via a WebSocket connection.
+    Client for handling audio transcription tasks via a single WebSocket connection.
 
-    Acts as a high-level client for audio transcription tasks using a WebSocket connection. It can be used
+    Acts as a high-level client for audio transcription tasksoutput_transcription_path using a WebSocket connection. It can be used
     to send audio data for transcription to a server and receive transcribed text segments.
 
     Args:
         host (str): The hostname or IP address of the server.
         port (int): The port number to connect to on the server.
         lang (str, optional): The primary language for transcription. Default is None, which defaults to English ('en').
-        translate (bool, optional): Indicates whether translation tasks are required (default is False).
+        translate (bool, optional): If True, the task will be translation instead of transcription. Default is False.
+        model (str, optional): The whisper model to use (e.g., "small", "base"). Default is "small".
+        use_vad (bool, optional): Whether to enable voice activity detection. Default is True.
+        save_output_recording (bool, optional): Whether to save the microphone recording. Default is False.
+        output_recording_filename (str, optional): Path to save the output recording WAV file. Default is "./output_recording.wav".
+        output_transcription_path (str, optional): File path to save the output transcription (SRT file). Default is "./output.srt".
+        log_transcription (bool, optional): Whether to log transcription output to the console. Default is True.
+        mute_audio_playback (bool, optional): If True, mutes audio playback during file playback. Default is False.
+        send_last_n_segments (int, optional): Number of most recent segments to send to the client. Defaults to 10.
+        no_speech_thresh (float, optional): Segments with no speech probability above this threshold will be discarded. Defaults to 0.45.
+        clip_audio (bool, optional): Whether to clip audio with no valid segments. Defaults to False.
+        same_output_threshold (int, optional): Number of repeated outputs before considering it as a valid segment. Defaults to 10.
+        transcription_callback (callable, optional): A callback function to handle transcription results. Default is None.
+        enable_translation (float, optional): Whether to enable translation from any to any language. Defaults to False.
+        target_language (str, optional): Target language for translation. Defaults to 'fr'.
+        translation_callback (callable, optional): A callback function to handle translation results. Default is None.
+        translation_srt_file_path (str, optional): The file path to save the translated output SRT file. Default is "output_translated.srt".
 
     Attributes:
         client (Client): An instance of the underlying Client class responsible for handling the WebSocket connection.
@@ -506,32 +795,65 @@ class TranscriptionClient:
         transcription_client()
         ```
     """
-    def __init__(self, host, port, lang=None, translate=False, model="small", use_vad=True):
-        self.client = Client(host, port, lang, translate, model, srt_file_path="output.srt", use_vad=use_vad)
+    def __init__(
+        self,
+        host,
+        port,
+        lang=None,
+        translate=False,
+        model="small",
+        use_vad=True,
+        use_wss=False,
+        save_output_recording=False,
+        output_recording_filename="./output_recording.wav",
+        output_transcription_path="./output.srt",
+        log_transcription=True,
+        mute_audio_playback=False,
+        send_last_n_segments=10,
+        no_speech_thresh=0.45,
+        clip_audio=False,
+        same_output_threshold=10,
+        transcription_callback=None,
+        enable_translation=False,
+        target_language="fr",
+        translation_callback=None,
+        translation_srt_file_path="./output_translated.srt",
+        enable_timestamps=False,
+        display_segments=4,
+    ):
+        self.client = Client(
+            host,
+            port,
+            lang,
+            translate,
+            model,
+            srt_file_path=output_transcription_path,
+            use_vad=use_vad,
+            use_wss=use_wss,
+            log_transcription=log_transcription,
+            send_last_n_segments=send_last_n_segments,
+            no_speech_thresh=no_speech_thresh,
+            clip_audio=clip_audio,
+            same_output_threshold=same_output_threshold,
+            transcription_callback=transcription_callback,
+            enable_translation=enable_translation,
+            target_language=target_language,
+            translation_callback=translation_callback,
+            translation_srt_file_path=translation_srt_file_path,
+            enable_timestamps=enable_timestamps,
+            display_segments=display_segments,
+        )
 
-    def __call__(self, audio=None, hls_url=None):
-        """
-        Start the transcription process.
-
-        Initiates the transcription process by connecting to the server via a WebSocket. It waits for the server
-        to be ready to receive audio data and then sends audio for transcription. If an audio file is provided, it
-        will be played and streamed to the server; otherwise, it will perform live recording.
-
-        Args:
-            audio (str, optional): Path to an audio file for transcription. Default is None, which triggers live recording.
-
-        """
-        print("[INFO]: Waiting for server ready ...")
-        while not self.client.recording:
-            if self.client.waiting or self.client.server_error:
-                self.client.close_websocket()
-                return
-
-        print("[INFO]: Server Ready!")
-        if hls_url is not None:
-            self.client.process_hls_stream(hls_url)
-        elif audio is not None:
-            resampled_file = utils.resample(audio)
-            self.client.play_file(resampled_file)
-        else:
-            self.client.record()
+        if save_output_recording and not output_recording_filename.endswith(".wav"):
+            raise ValueError(f"Please provide a valid `output_recording_filename`: {output_recording_filename}")
+        if not output_transcription_path.endswith(".srt"):
+            raise ValueError(f"Please provide a valid `output_transcription_path`: {output_transcription_path}. The file extension should be `.srt`.")
+        if not translation_srt_file_path.endswith(".srt"):
+            raise ValueError(f"Please provide a valid `translation_srt_file_path`: {translation_srt_file_path}. The file extension should be `.srt`.")
+        TranscriptionTeeClient.__init__(
+            self,
+            [self.client],
+            save_output_recording=save_output_recording,
+            output_recording_filename=output_recording_filename,
+            mute_audio_playback=mute_audio_playback
+        )

whisper_live/transcriber/tensorrt_utils.py

@@ -23,8 +23,12 @@ from typing import Dict, Iterable, List, Optional, TextIO, Tuple, Union
 import kaldialign
 import numpy as np
 import soundfile
+import av
+import wave
 import torch
 import torch.nn.functional as F
+from whisper_live.utils import resample
+
 
 Pathlike = Union[str, Path]
 
@@ -35,38 +39,33 @@ CHUNK_LENGTH = 30
 N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE  # 480000 samples in a 30-second chunk
 
 
-def load_audio(file: str, sr: int = SAMPLE_RATE):
+def load_audio(file: str, sr: int = 16000):
     """
-    Open an audio file and read as mono waveform, resampling as necessary
+    Open an audio file, resample it, and read as a mono waveform.
 
     Parameters
     ----------
     file: str
-        The audio file to open
+        The audio file to open.
 
     sr: int
-        The sample rate to resample the audio if necessary
+        The sample rate to resample the audio if necessary.
 
     Returns
     -------
     A NumPy array containing the audio waveform, in float32 dtype.
     """
+    resampled_file = resample(file, sr)
 
-    # This launches a subprocess to decode audio while down-mixing
-    # and resampling as necessary.  Requires the ffmpeg CLI in PATH.
-    # fmt: off
-    cmd = [
-        "ffmpeg", "-nostdin", "-threads", "0", "-i", file, "-f", "s16le", "-ac",
-        "1", "-acodec", "pcm_s16le", "-ar",
-        str(sr), "-"
-    ]
-    # fmt: on
-    try:
-        out = run(cmd, capture_output=True, check=True).stdout
-    except CalledProcessError as e:
-        raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e
+    with wave.open(resampled_file, "rb") as wav_file:
+        num_frames = wav_file.getnframes()
+        raw_data = wav_file.readframes(num_frames)
 
-    return np.frombuffer(out, np.int16).flatten().astype(np.float32) / 32768.0
+        audio_data = np.frombuffer(raw_data, dtype=np.int16)
+
+    audio_data = audio_data.astype(np.float32) / 32768.0
+
+    return audio_data
 
 
 def load_audio_wav_format(wav_path):

whisper_live/transcriber/transcriber_openvino.py

@@ -0,0 +1,23 @@
+import librosa
+import os
+
+import openvino_genai as ov_genai
+import huggingface_hub as hf_hub
+
+
+class WhisperOpenVINO(object):
+    def __init__(self, model_id="OpenVINO/whisper-tiny-fp16-ov", device="CPU", language="en", task="transcribe"):
+        model_path = model_id.split('/')[-1]
+        cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "openvino_whisper_models")
+        os.makedirs(cache_dir, exist_ok=True)
+        model_path = os.path.join(cache_dir, model_path)
+        if not os.path.exists(model_path):
+            hf_hub.snapshot_download(model_id, local_dir=model_path)
+        self.model = ov_genai.WhisperPipeline(str(model_path), device=device)
+        self.language = language
+        self.task = task
+
+    def transcribe(self, input_audio):
+        outputs = self.model.generate(input_audio, return_timestamps=True, language=self.language, task=self.task)
+        outputs = [seg for seg in outputs.chunks]
+        return outputs

whisper_live/transcriber/transcriber_tensorrt.py

@@ -0,0 +1,479 @@
+import json
+import re
+import math
+from collections import OrderedDict
+from pathlib import Path
+from typing import Union
+
+import torch
+import numpy as np
+import torch.nn.functional as F
+from whisper.tokenizer import get_tokenizer
+from whisper_live.transcriber.tensorrt_utils import (
+    mel_filters, 
+    load_audio_wav_format, 
+    pad_or_trim, 
+    load_audio
+)
+
+import tensorrt_llm
+import tensorrt_llm.logger as logger
+from tensorrt_llm._utils import (str_dtype_to_torch, str_dtype_to_trt,
+                                 trt_dtype_to_torch)
+from tensorrt_llm.bindings import GptJsonConfig, KVCacheType
+from tensorrt_llm.runtime import PYTHON_BINDINGS, ModelConfig, SamplingConfig
+from tensorrt_llm.runtime.session import Session, TensorInfo
+if PYTHON_BINDINGS:
+    from tensorrt_llm.runtime import ModelRunnerCpp
+
+SAMPLE_RATE = 16000
+N_FFT = 400
+HOP_LENGTH = 160
+CHUNK_LENGTH = 30
+N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE  # 480000 samples in a 30-second chunk
+
+def read_config(component, engine_dir):
+    config_path = engine_dir / component / 'config.json'
+    with open(config_path, 'r') as f:
+        config = json.load(f)
+    model_config = OrderedDict()
+    model_config.update(config['pretrained_config'])
+    model_config.update(config['build_config'])
+    return model_config
+
+
+def remove_tensor_padding(input_tensor,
+                          input_tensor_lengths=None,
+                          pad_value=None):
+    if pad_value:
+        assert input_tensor_lengths is None, "input_tensor_lengths should be None when pad_value is provided"
+        # Text tensor case: batch, seq_len
+        assert torch.all(
+            input_tensor[:, 0] != pad_value
+        ), "First token in each sequence should not be pad_value"
+        assert input_tensor_lengths is None
+
+        # Create a mask for all non-pad tokens
+        mask = input_tensor != pad_value
+
+        # Apply the mask to input_tensor to remove pad tokens
+        output_tensor = input_tensor[mask].view(1, -1)
+
+    else:
+        # Audio tensor case: batch, seq_len, feature_len
+        # position_ids case: batch, seq_len
+        assert input_tensor_lengths is not None, "input_tensor_lengths must be provided for 3D input_tensor"
+
+        # Initialize a list to collect valid sequences
+        valid_sequences = []
+
+        for i in range(input_tensor.shape[0]):
+            valid_length = input_tensor_lengths[i]
+            valid_sequences.append(input_tensor[i, :valid_length])
+
+        # Concatenate all valid sequences along the batch dimension
+        output_tensor = torch.cat(valid_sequences, dim=0)
+    return output_tensor
+
+
+class WhisperEncoding:
+
+    def __init__(self, engine_dir):
+        self.session = self.get_session(engine_dir)
+        config = read_config('encoder', engine_dir)
+        self.n_mels = config['n_mels']
+        self.dtype = config['dtype']
+        self.num_languages = config['num_languages']
+        self.encoder_config = config
+
+    def get_session(self, engine_dir):
+        serialize_path = engine_dir / 'encoder' / 'rank0.engine'
+        with open(serialize_path, 'rb') as f:
+            session = Session.from_serialized_engine(f.read())
+        return session
+
+    def get_audio_features(self,
+                           mel,
+                           mel_input_lengths,
+                           encoder_downsampling_factor=2):
+        if isinstance(mel, list):
+            longest_mel = max([f.shape[-1] for f in mel])
+            mel = [
+                torch.nn.functional.pad(f, (0, longest_mel - f.shape[-1]),
+                                        mode='constant') for f in mel
+            ]
+            mel = torch.cat(mel, dim=0).type(
+                str_dtype_to_torch("float16")).contiguous()
+        bsz, seq_len = mel.shape[0], mel.shape[2]
+        position_ids = torch.arange(
+            math.ceil(seq_len / encoder_downsampling_factor),
+            dtype=torch.int32,
+            device=mel.device).expand(bsz, -1).contiguous()
+        if self.encoder_config['plugin_config']['remove_input_padding']:
+            # mel B,D,T -> B,T,D -> BxT, D
+            mel = mel.transpose(1, 2)
+            mel = remove_tensor_padding(mel, mel_input_lengths)
+            position_ids = remove_tensor_padding(
+                position_ids, mel_input_lengths // encoder_downsampling_factor)
+        inputs = OrderedDict()
+        inputs['input_features'] = mel
+        inputs['input_lengths'] = mel_input_lengths
+        inputs['position_ids'] = position_ids
+
+        output_list = [
+            TensorInfo('input_features', str_dtype_to_trt(self.dtype),
+                       mel.shape),
+            TensorInfo('input_lengths', str_dtype_to_trt('int32'),
+                       mel_input_lengths.shape),
+            TensorInfo('position_ids', str_dtype_to_trt('int32'),
+                       inputs['position_ids'].shape)
+        ]
+
+        output_info = (self.session).infer_shapes(output_list)
+
+        logger.debug(f'output info {output_info}')
+        outputs = {
+            t.name: torch.empty(tuple(t.shape),
+                                dtype=trt_dtype_to_torch(t.dtype),
+                                device='cuda')
+            for t in output_info
+        }
+        stream = torch.cuda.current_stream()
+        ok = self.session.run(inputs=inputs,
+                              outputs=outputs,
+                              stream=stream.cuda_stream)
+        assert ok, 'Engine execution failed'
+        stream.synchronize()
+        encoder_output = outputs['encoder_output']
+        encoder_output_lengths = mel_input_lengths // encoder_downsampling_factor
+        return encoder_output, encoder_output_lengths
+
+
+class WhisperDecoding:
+
+    def __init__(self, engine_dir, runtime_mapping, debug_mode=False):
+
+        self.decoder_config = read_config('decoder', engine_dir)
+        self.decoder_generation_session = self.get_session(
+            engine_dir, runtime_mapping, debug_mode)
+
+    def get_session(self, engine_dir, runtime_mapping, debug_mode=False):
+        serialize_path = engine_dir / 'decoder' / 'rank0.engine'
+        with open(serialize_path, "rb") as f:
+            decoder_engine_buffer = f.read()
+
+        decoder_model_config = ModelConfig(
+            max_batch_size=self.decoder_config['max_batch_size'],
+            max_beam_width=self.decoder_config['max_beam_width'],
+            num_heads=self.decoder_config['num_attention_heads'],
+            num_kv_heads=self.decoder_config['num_attention_heads'],
+            hidden_size=self.decoder_config['hidden_size'],
+            vocab_size=self.decoder_config['vocab_size'],
+            cross_attention=True,
+            num_layers=self.decoder_config['num_hidden_layers'],
+            gpt_attention_plugin=self.decoder_config['plugin_config']
+            ['gpt_attention_plugin'],
+            remove_input_padding=self.decoder_config['plugin_config']
+            ['remove_input_padding'],
+            kv_cache_type=KVCacheType.PAGED
+            if self.decoder_config['plugin_config']['paged_kv_cache'] == True
+            else KVCacheType.CONTINUOUS,
+            has_position_embedding=self.
+            decoder_config['has_position_embedding'],
+            dtype=self.decoder_config['dtype'],
+            has_token_type_embedding=False,
+        )
+        decoder_generation_session = tensorrt_llm.runtime.GenerationSession(
+            decoder_model_config,
+            decoder_engine_buffer,
+            runtime_mapping,
+            debug_mode=debug_mode)
+
+        return decoder_generation_session
+
+    def generate(self,
+                 decoder_input_ids,
+                 encoder_outputs,
+                 encoder_max_input_length,
+                 encoder_input_lengths,
+                 eot_id,
+                 max_new_tokens=40,
+                 num_beams=1):
+        batch_size = decoder_input_ids.shape[0]
+        decoder_input_lengths = torch.tensor([
+            decoder_input_ids.shape[-1]
+            for _ in range(decoder_input_ids.shape[0])
+        ],
+                                             dtype=torch.int32,
+                                             device='cuda')
+        decoder_max_input_length = torch.max(decoder_input_lengths).item()
+
+        cross_attention_mask = torch.ones([
+            batch_size, decoder_max_input_length + max_new_tokens,
+            encoder_max_input_length
+        ]).int().cuda()
+        # generation config
+        sampling_config = SamplingConfig(end_id=eot_id,
+                                         pad_id=eot_id,
+                                         num_beams=num_beams)
+        self.decoder_generation_session.setup(
+            decoder_input_lengths.size(0),
+            decoder_max_input_length,
+            max_new_tokens,
+            beam_width=num_beams,
+            encoder_max_input_length=encoder_max_input_length)
+
+        torch.cuda.synchronize()
+
+        decoder_input_ids = decoder_input_ids.type(torch.int32).cuda()
+        if self.decoder_config['plugin_config']['remove_input_padding']:
+            # 50256 is the index of <pad> for all whisper models' decoder
+            WHISPER_PAD_TOKEN_ID = 50256
+            decoder_input_ids = remove_tensor_padding(
+                decoder_input_ids, pad_value=WHISPER_PAD_TOKEN_ID)
+            if encoder_outputs.dim() == 3:
+                encoder_output_lens = torch.full((encoder_outputs.shape[0], ),
+                                                 encoder_outputs.shape[1],
+                                                 dtype=torch.int32,
+                                                 device='cuda')
+
+                encoder_outputs = remove_tensor_padding(encoder_outputs,
+                                                        encoder_output_lens)
+        output_ids = self.decoder_generation_session.decode(
+            decoder_input_ids,
+            decoder_input_lengths,
+            sampling_config,
+            encoder_output=encoder_outputs,
+            encoder_input_lengths=encoder_input_lengths,
+            cross_attention_mask=cross_attention_mask,
+        )
+        torch.cuda.synchronize()
+
+        # get the list of int from output_ids tensor
+        output_ids = output_ids.cpu().numpy().tolist()
+        return output_ids
+
+
+class WhisperTRTLLM(object):
+
+    def __init__(self, 
+                 engine_dir,
+                 assets_dir=None,
+                 device=None,
+                 is_multilingual=False,
+                 language="en",
+                 task="transcribe",
+                 use_py_session=False,
+                 num_beams=1,
+                 debug_mode=False,
+                 max_output_len=96):
+        world_size = 1
+        runtime_rank = tensorrt_llm.mpi_rank()
+        runtime_mapping = tensorrt_llm.Mapping(world_size, runtime_rank)
+        torch.cuda.set_device(runtime_rank % runtime_mapping.gpus_per_node)
+        engine_dir = Path(engine_dir)
+        encoder_config = read_config('encoder', engine_dir)
+        decoder_config = read_config('decoder', engine_dir)
+        self.n_mels = encoder_config['n_mels']
+        self.num_languages = encoder_config['num_languages']
+        is_multilingual = (decoder_config['vocab_size'] >= 51865)
+
+        self.device = device
+        self.tokenizer = get_tokenizer(
+            is_multilingual,
+            num_languages=self.num_languages,
+            language=language,
+            task=task,
+        )
+
+        if use_py_session:
+            self.encoder = WhisperEncoding(engine_dir)
+            self.decoder = WhisperDecoding(engine_dir,
+                                        runtime_mapping,
+                                        debug_mode=False)
+        else:
+            json_config = GptJsonConfig.parse_file(engine_dir / 'decoder' /
+                                                   'config.json')
+            assert json_config.model_config.supports_inflight_batching
+            runner_kwargs = dict(engine_dir=engine_dir,
+                                 is_enc_dec=True,
+                                 max_batch_size=1,
+                                 max_input_len=3000,
+                                 max_output_len=max_output_len,
+                                 max_beam_width=num_beams,
+                                 debug_mode=debug_mode,
+                                 kv_cache_free_gpu_memory_fraction=0.9,
+                                 cross_kv_cache_fraction=0.5)
+            self.model_runner_cpp = ModelRunnerCpp.from_dir(**runner_kwargs)
+        self.filters = mel_filters(self.device, self.n_mels, assets_dir)
+        self.use_py_session = use_py_session
+
+    def log_mel_spectrogram(
+        self,
+        audio: Union[str, np.ndarray, torch.Tensor],
+        padding: int = 0,
+        return_duration=True
+    ):
+        """
+        Compute the log-Mel spectrogram of
+
+        Parameters
+        ----------
+        audio: Union[str, np.ndarray, torch.Tensor], shape = (*)
+            The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz
+
+        n_mels: int
+            The number of Mel-frequency filters, only 80 and 128 are supported
+
+        padding: int
+            Number of zero samples to pad to the right
+
+        device: Optional[Union[str, torch.device]]
+            If given, the audio tensor is moved to this device before STFT
+
+        Returns
+        -------
+        torch.Tensor, shape = (80 or 128, n_frames)
+            A Tensor that contains the Mel spectrogram
+        """
+        if not torch.is_tensor(audio):
+            if isinstance(audio, str):
+                if audio.endswith('.wav'):
+                    audio, _ = load_audio_wav_format(audio)
+                else:
+                    audio = load_audio(audio)
+            assert isinstance(audio, np.ndarray), f"Unsupported audio type: {type(audio)}"
+            duration = audio.shape[-1] / SAMPLE_RATE
+            audio = pad_or_trim(audio, N_SAMPLES)
+            audio = audio.astype(np.float32)
+            audio = torch.from_numpy(audio)
+
+        if self.device is not None:
+            audio = audio.to(self.device)
+        if padding > 0:
+            audio = F.pad(audio, (0, padding))
+        window = torch.hann_window(N_FFT).to(audio.device)
+        stft = torch.stft(audio, N_FFT, HOP_LENGTH, window=window, return_complex=True)
+        magnitudes = stft[..., :-1].abs()**2
+
+        mel_spec = self.filters @ magnitudes
+
+        log_spec = torch.clamp(mel_spec, min=1e-10).log10()
+        log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
+        log_spec = (log_spec + 4.0) / 4.0
+        if return_duration:
+            return log_spec, duration
+        else:
+            return log_spec
+
+    def process_batch(
+            self,
+            mel,
+            mel_input_lengths,
+            text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
+            num_beams=1,
+            max_new_tokens=96):
+        prompt_id = self.tokenizer.encode(
+            text_prefix, allowed_special=set(self.tokenizer.special_tokens.keys()))
+
+        prompt_id = torch.tensor(prompt_id)
+        batch_size = mel.shape[0]
+        decoder_input_ids = prompt_id.repeat(batch_size, 1)
+        if self.use_py_session:
+            encoder_output, encoder_output_lengths = self.encoder.get_audio_features(mel, mel_input_lengths)
+            encoder_max_input_length = torch.max(encoder_output_lengths).item()
+            output_ids = self.decoder.generate(decoder_input_ids,
+                                            encoder_output,
+                                            encoder_max_input_length,
+                                            encoder_output_lengths,
+                                            self.tokenizer.eot,
+                                            max_new_tokens=max_new_tokens,
+                                            num_beams=num_beams)
+        else:
+            with torch.no_grad():
+                if isinstance(mel, list):
+                    mel = [
+                        m.transpose(1, 2).type(
+                            str_dtype_to_torch("float16")).squeeze(0)
+                        for m in mel
+                    ]
+                else:
+                    mel = mel.transpose(1, 2)
+                outputs = self.model_runner_cpp.generate(
+                    batch_input_ids=decoder_input_ids,
+                    encoder_input_features=mel,
+                    encoder_output_lengths=mel_input_lengths // 2,
+                    max_new_tokens=max_new_tokens,
+                    end_id=self.tokenizer.eot,
+                    pad_id=self.tokenizer.eot,
+                    num_beams=num_beams,
+                    output_sequence_lengths=True,
+                    return_dict=True)
+                torch.cuda.synchronize()
+                output_ids = outputs['output_ids'].cpu().numpy().tolist()
+        texts = []
+        for i in range(len(output_ids)):
+            text = self.tokenizer.decode(output_ids[i][0]).strip()
+            texts.append(text)
+        return texts
+
+    def transcribe(
+            self,
+            mel,
+            text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
+            dtype='float16',
+            batch_size=1,
+            num_beams=1,
+            padding_strategy="max",
+            max_new_tokens=96,
+        ):
+        mel = mel.type(str_dtype_to_torch(dtype))
+        mel = mel.unsqueeze(0)
+        # repeat the mel spectrogram to match the batch size
+        mel = mel.repeat(batch_size, 1, 1)
+        if padding_strategy == "longest":
+            pass
+        else:
+            mel = torch.nn.functional.pad(mel, (0, 3000 - mel.shape[2]))
+        features_input_lengths = torch.full((mel.shape[0], ),
+                                             mel.shape[2],
+                                             dtype=torch.int32,
+                                             device=mel.device)
+
+        predictions = self.process_batch(
+            mel,
+            features_input_lengths,
+            text_prefix,
+            num_beams,
+            max_new_tokens=max_new_tokens
+        )
+        prediction = predictions[0]
+
+        # remove all special tokens in the prediction
+        prediction = re.sub(r'<\|.*?\|>', '', prediction)
+        return prediction.strip()
+
+
+def decode_wav_file(
+        model,
+        mel,
+        text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
+        dtype='float16',
+        batch_size=1,
+        num_beams=1,
+        normalizer=None,
+        mel_filters_dir=None):
+
+    mel = mel.type(str_dtype_to_torch(dtype))
+    mel = mel.unsqueeze(0)
+    # repeat the mel spectrogram to match the batch size
+    mel = mel.repeat(batch_size, 1, 1)
+    predictions = model.process_batch(mel, text_prefix, num_beams)
+    prediction = predictions[0]
+
+    # remove all special tokens in the prediction
+    prediction = re.sub(r'<\|.*?\|>', '', prediction)
+    if normalizer:
+        prediction = normalizer(prediction)
+
+    return prediction.strip()

whisper_live/transcriber_tensorrt.py

@@ -1,320 +0,0 @@
-import json
-import re
-from collections import OrderedDict
-from pathlib import Path
-from typing import Union
-
-import torch
-import numpy as np
-import torch.nn.functional as F
-from whisper.tokenizer import get_tokenizer
-from whisper_live.tensorrt_utils import (mel_filters, load_audio_wav_format, pad_or_trim, load_audio)
-
-import tensorrt_llm
-import tensorrt_llm.logger as logger
-from tensorrt_llm._utils import (str_dtype_to_torch, str_dtype_to_trt,
-                                 trt_dtype_to_torch)
-from tensorrt_llm.runtime import ModelConfig, SamplingConfig
-from tensorrt_llm.runtime.session import Session, TensorInfo
-
-
-SAMPLE_RATE = 16000
-N_FFT = 400
-HOP_LENGTH = 160
-CHUNK_LENGTH = 30
-N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE  # 480000 samples in a 30-second chunk
-
-
-class WhisperEncoding:
-
-    def __init__(self, engine_dir):
-        self.session = self.get_session(engine_dir)
-
-    def get_session(self, engine_dir):
-        config_path = engine_dir / 'encoder_config.json'
-        with open(config_path, 'r') as f:
-            config = json.load(f)
-
-        dtype = config['builder_config']['precision']
-        n_mels = config['builder_config']['n_mels']
-        num_languages = config['builder_config']['num_languages']
-
-        self.dtype = dtype
-        self.n_mels = n_mels
-        self.num_languages = num_languages
-
-        serialize_path = engine_dir / f'whisper_encoder_{self.dtype}_tp1_rank0.engine'
-
-        with open(serialize_path, 'rb') as f:
-            session = Session.from_serialized_engine(f.read())
-
-        return session
-
-    def get_audio_features(self, mel):
-        inputs = OrderedDict()
-        output_list = []
-
-        inputs.update({'x': mel})
-        output_list.append(
-            TensorInfo('x', str_dtype_to_trt(self.dtype), mel.shape))
-
-        output_info = (self.session).infer_shapes(output_list)
-
-        logger.debug(f'output info {output_info}')
-        outputs = {
-            t.name: torch.empty(tuple(t.shape),
-                                dtype=trt_dtype_to_torch(t.dtype),
-                                device='cuda')
-            for t in output_info
-        }
-        stream = torch.cuda.current_stream()
-        ok = self.session.run(inputs=inputs,
-                              outputs=outputs,
-                              stream=stream.cuda_stream)
-        assert ok, 'Engine execution failed'
-        stream.synchronize()
-        audio_features = outputs['output']
-        return audio_features
-
-
-class WhisperDecoding:
-
-    def __init__(self, engine_dir, runtime_mapping, debug_mode=False):
-
-        self.decoder_config = self.get_config(engine_dir)
-        self.decoder_generation_session = self.get_session(
-            engine_dir, runtime_mapping, debug_mode)
-
-    def get_config(self, engine_dir):
-        config_path = engine_dir / 'decoder_config.json'
-        with open(config_path, 'r') as f:
-            config = json.load(f)
-        decoder_config = OrderedDict()
-        decoder_config.update(config['plugin_config'])
-        decoder_config.update(config['builder_config'])
-        return decoder_config
-
-    def get_session(self, engine_dir, runtime_mapping, debug_mode=False):
-        dtype = self.decoder_config['precision']
-        serialize_path = engine_dir / f'whisper_decoder_{dtype}_tp1_rank0.engine'
-        with open(serialize_path, "rb") as f:
-            decoder_engine_buffer = f.read()
-
-        decoder_model_config = ModelConfig(
-            num_heads=self.decoder_config['num_heads'],
-            num_kv_heads=self.decoder_config['num_heads'],
-            hidden_size=self.decoder_config['hidden_size'],
-            vocab_size=self.decoder_config['vocab_size'],
-            num_layers=self.decoder_config['num_layers'],
-            gpt_attention_plugin=self.decoder_config['gpt_attention_plugin'],
-            remove_input_padding=self.decoder_config['remove_input_padding'],
-            cross_attention=self.decoder_config['cross_attention'],
-            has_position_embedding=self.
-            decoder_config['has_position_embedding'],
-            has_token_type_embedding=self.
-            decoder_config['has_token_type_embedding'],
-        )
-        decoder_generation_session = tensorrt_llm.runtime.GenerationSession(
-            decoder_model_config,
-            decoder_engine_buffer,
-            runtime_mapping,
-            debug_mode=debug_mode)
-
-        return decoder_generation_session
-
-    def generate(self,
-                 decoder_input_ids,
-                 encoder_outputs,
-                 eot_id,
-                 max_new_tokens=40,
-                 num_beams=1):
-        encoder_input_lengths = torch.tensor(
-            [encoder_outputs.shape[1] for x in range(encoder_outputs.shape[0])],
-            dtype=torch.int32,
-            device='cuda')
-
-        decoder_input_lengths = torch.tensor([
-            decoder_input_ids.shape[-1]
-            for _ in range(decoder_input_ids.shape[0])
-        ],
-                                             dtype=torch.int32,
-                                             device='cuda')
-        decoder_max_input_length = torch.max(decoder_input_lengths).item()
-
-        # generation config
-        sampling_config = SamplingConfig(end_id=eot_id,
-                                         pad_id=eot_id,
-                                         num_beams=num_beams)
-        self.decoder_generation_session.setup(
-            decoder_input_lengths.size(0),
-            decoder_max_input_length,
-            max_new_tokens,
-            beam_width=num_beams,
-            encoder_max_input_length=encoder_outputs.shape[1])
-
-        torch.cuda.synchronize()
-
-        decoder_input_ids = decoder_input_ids.type(torch.int32).cuda()
-        output_ids = self.decoder_generation_session.decode(
-            decoder_input_ids,
-            decoder_input_lengths,
-            sampling_config,
-            encoder_output=encoder_outputs,
-            encoder_input_lengths=encoder_input_lengths,
-        )
-        torch.cuda.synchronize()
-
-        # get the list of int from output_ids tensor
-        output_ids = output_ids.cpu().numpy().tolist()
-        return output_ids
-
-
-class WhisperTRTLLM(object):
-
-    def __init__(self, engine_dir, assets_dir=None, device=None, is_multilingual=False,
-                 language="en", task="transcribe"):
-        world_size = 1
-        runtime_rank = tensorrt_llm.mpi_rank()
-        runtime_mapping = tensorrt_llm.Mapping(world_size, runtime_rank)
-        torch.cuda.set_device(runtime_rank % runtime_mapping.gpus_per_node)
-        engine_dir = Path(engine_dir)
-
-        self.encoder = WhisperEncoding(engine_dir)
-        self.decoder = WhisperDecoding(engine_dir,
-                                       runtime_mapping,
-                                       debug_mode=False)
-        self.n_mels = self.encoder.n_mels
-        # self.tokenizer = get_tokenizer(num_languages=self.encoder.num_languages,
-        #                                tokenizer_dir=assets_dir)
-        self.device = device
-        self.tokenizer = get_tokenizer(
-            is_multilingual,
-            num_languages=self.encoder.num_languages,
-            language=language,
-            task=task,
-        )
-        self.filters = mel_filters(self.device, self.encoder.n_mels, assets_dir)
-
-    def log_mel_spectrogram(
-        self,
-        audio: Union[str, np.ndarray, torch.Tensor],
-        padding: int = 0,
-        return_duration=True
-    ):
-        """
-        Compute the log-Mel spectrogram of
-
-        Parameters
-        ----------
-        audio: Union[str, np.ndarray, torch.Tensor], shape = (*)
-            The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz
-
-        n_mels: int
-            The number of Mel-frequency filters, only 80 and 128 are supported
-
-        padding: int
-            Number of zero samples to pad to the right
-
-        device: Optional[Union[str, torch.device]]
-            If given, the audio tensor is moved to this device before STFT
-
-        Returns
-        -------
-        torch.Tensor, shape = (80 or 128, n_frames)
-            A Tensor that contains the Mel spectrogram
-        """
-        if not torch.is_tensor(audio):
-            if isinstance(audio, str):
-                if audio.endswith('.wav'):
-                    audio, _ = load_audio_wav_format(audio)
-                else:
-                    audio = load_audio(audio)
-            assert isinstance(audio, np.ndarray), f"Unsupported audio type: {type(audio)}"
-            duration = audio.shape[-1] / SAMPLE_RATE
-            audio = pad_or_trim(audio, N_SAMPLES)
-            audio = audio.astype(np.float32)
-            audio = torch.from_numpy(audio)
-
-        if self.device is not None:
-            audio = audio.to(self.device)
-        if padding > 0:
-            audio = F.pad(audio, (0, padding))
-        window = torch.hann_window(N_FFT).to(audio.device)
-        stft = torch.stft(audio, N_FFT, HOP_LENGTH, window=window, return_complex=True)
-        magnitudes = stft[..., :-1].abs()**2
-
-        mel_spec = self.filters @ magnitudes
-
-        log_spec = torch.clamp(mel_spec, min=1e-10).log10()
-        log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)
-        log_spec = (log_spec + 4.0) / 4.0
-        if return_duration:
-            return log_spec, duration
-        else:
-            return log_spec
-
-    def process_batch(
-            self,
-            mel,
-            text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
-            num_beams=1):
-        prompt_id = self.tokenizer.encode(
-            text_prefix, allowed_special=set(self.tokenizer.special_tokens.keys()))
-
-        prompt_id = torch.tensor(prompt_id)
-        batch_size = mel.shape[0]
-        decoder_input_ids = prompt_id.repeat(batch_size, 1)
-
-        encoder_output = self.encoder.get_audio_features(mel)
-        output_ids = self.decoder.generate(decoder_input_ids,
-                                           encoder_output,
-                                           self.tokenizer.eot,
-                                           max_new_tokens=96,
-                                           num_beams=num_beams)
-        texts = []
-        for i in range(len(output_ids)):
-            text = self.tokenizer.decode(output_ids[i][0]).strip()
-            texts.append(text)
-        return texts
-
-    def transcribe(
-            self,
-            mel,
-            text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
-            dtype='float16',
-            batch_size=1,
-            num_beams=1,
-            ):
-        mel = mel.type(str_dtype_to_torch(dtype))
-        mel = mel.unsqueeze(0)
-        predictions = self.process_batch(mel, text_prefix, num_beams)
-        prediction = predictions[0]
-
-        # remove all special tokens in the prediction
-        prediction = re.sub(r'<\|.*?\|>', '', prediction)
-        return prediction.strip()
-
-
-def decode_wav_file(
-        model,
-        mel,
-        text_prefix="<|startoftranscript|><|en|><|transcribe|><|notimestamps|>",
-        dtype='float16',
-        batch_size=1,
-        num_beams=1,
-        normalizer=None,
-        mel_filters_dir=None):
-
-    mel = mel.type(str_dtype_to_torch(dtype))
-    mel = mel.unsqueeze(0)
-    # repeat the mel spectrogram to match the batch size
-    mel = mel.repeat(batch_size, 1, 1)
-    predictions = model.process_batch(mel, text_prefix, num_beams)
-    prediction = predictions[0]
-
-    # remove all special tokens in the prediction
-    prediction = re.sub(r'<\|.*?\|>', '', prediction)
-    if normalizer:
-        prediction = normalizer(prediction)
-
-    return prediction.strip()

whisper_live/utils.py

@@ -1,8 +1,9 @@
 import os
 import textwrap
 import scipy
-import ffmpeg
 import numpy as np
+import av
+from pathlib import Path
 
 
 def clear_screen():
@@ -10,11 +11,16 @@ def clear_screen():
     os.system("cls" if os.name == "nt" else "clear")
 
 
-def print_transcript(text):
+def print_transcript(text, translated=False, timestamps=False):
     """Prints formatted transcript text."""
-    wrapper = textwrap.TextWrapper(width=60)
-    for line in wrapper.wrap(text="".join(text)):
-        print(line)
+    if timestamps:
+        for t in text:
+            print(f'[{t["start"]} -> {t["end"]}] {t["text"]}')
+    else:
+        wrapper = textwrap.TextWrapper(width=60)
+        text=" ".join(text) if translated else "".join(text)
+        for line in wrapper.wrap(text=text):
+            print(line)
 
 
 def format_time(s):
@@ -26,8 +32,8 @@ def format_time(s):
     return f"{hours:02}:{minutes:02}:{seconds:02},{milliseconds:03}"
 
 
-def create_srt_file(segments, output_file):
-    with open(output_file, 'w', encoding='utf-8') as srt_file:
+def create_srt_file(segments, resampled_file):
+    with open(resampled_file, 'w', encoding='utf-8') as srt_file:
         segment_number = 1
         for segment in segments:
             start_time = format_time(float(segment['start']))
@@ -43,9 +49,7 @@ def create_srt_file(segments, output_file):
 
 def resample(file: str, sr: int = 16000):
     """
-    # https://github.com/openai/whisper/blob/7858aa9c08d98f75575035ecd6481f462d66ca27/whisper/audio.py#L22
-    Open an audio file and read as mono waveform, resampling as necessary,
-    save the resampled audio
+    Resample the audio file to 16kHz.
 
     Args:
         file (str): The audio file to open
@@ -54,18 +58,30 @@ def resample(file: str, sr: int = 16000):
     Returns:
         resampled_file (str): The resampled audio file
     """
-    try:
-        # This launches a subprocess to decode audio while down-mixing and resampling as necessary.
-        # Requires the ffmpeg CLI and `ffmpeg-python` package to be installed.
-        out, _ = (
-            ffmpeg.input(file, threads=0)
-            .output("-", format="s16le", acodec="pcm_s16le", ac=1, ar=sr)
-            .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True)
-        )
-    except ffmpeg.Error as e:
-        raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e
-    np_buffer = np.frombuffer(out, dtype=np.int16)
+    container = av.open(file)
+    stream = next(s for s in container.streams if s.type == 'audio')
 
-    resampled_file = f"{file.split('.')[0]}_resampled.wav"
-    scipy.io.wavfile.write(resampled_file, sr, np_buffer.astype(np.int16))
+    resampler = av.AudioResampler(
+        format='s16',
+        layout='mono',
+        rate=sr,
+    )
+
+    resampled_file = Path(file).stem + "_resampled.wav"
+    output_container = av.open(resampled_file, mode='w')
+    output_stream = output_container.add_stream('pcm_s16le', rate=sr)
+    output_stream.layout = 'mono'
+
+    for frame in container.decode(audio=0):
+        frame.pts = None
+        resampled_frames = resampler.resample(frame)
+        if resampled_frames is not None:
+            for resampled_frame in resampled_frames:
+                for packet in output_stream.encode(resampled_frame):
+                    output_container.mux(packet)
+
+    for packet in output_stream.encode(None):
+        output_container.mux(packet)
+
+    output_container.close()
     return resampled_file

whisper_live/vad.py

@@ -1,10 +1,9 @@
-# original: https://github.com/snakers4/silero-vad/blob/master/utils_vad.py
-
 import os
 import subprocess
 import torch
 import numpy as np
 import onnxruntime
+import warnings
 
 
 class VoiceActivityDetection():
@@ -24,7 +23,11 @@ class VoiceActivityDetection():
             self.session = onnxruntime.InferenceSession(path, providers=['CUDAExecutionProvider'], sess_options=opts)
 
         self.reset_states()
-        self.sample_rates = [8000, 16000]
+        if '16k' in path:
+            warnings.warn('This model support only 16000 sampling rate!')
+            self.sample_rates = [16000]
+        else:
+            self.sample_rates = [8000, 16000]
 
     def _validate_input(self, x, sr: int):
         if x.dim() == 1:
@@ -34,27 +37,32 @@ class VoiceActivityDetection():
 
         if sr != 16000 and (sr % 16000 == 0):
             step = sr // 16000
-            x = x[:, ::step]
+            x = x[:,::step]
             sr = 16000
 
         if sr not in self.sample_rates:
             raise ValueError(f"Supported sampling rates: {self.sample_rates} (or multiply of 16000)")
-
         if sr / x.shape[1] > 31.25:
             raise ValueError("Input audio chunk is too short")
 
         return x, sr
 
     def reset_states(self, batch_size=1):
-        self._h = np.zeros((2, batch_size, 64)).astype('float32')
-        self._c = np.zeros((2, batch_size, 64)).astype('float32')
+        self._state = torch.zeros((2, batch_size, 128)).float()
+        self._context = torch.zeros(0)
         self._last_sr = 0
         self._last_batch_size = 0
 
     def __call__(self, x, sr: int):
 
         x, sr = self._validate_input(x, sr)
+        num_samples = 512 if sr == 16000 else 256
+
+        if x.shape[-1] != num_samples:
+            raise ValueError(f"Provided number of samples is {x.shape[-1]} (Supported values: 256 for 8000 sample rate, 512 for 16000)")
+
         batch_size = x.shape[0]
+        context_size = 64 if sr == 16000 else 32
 
         if not self._last_batch_size:
             self.reset_states(batch_size)
@@ -63,28 +71,35 @@ class VoiceActivityDetection():
         if (self._last_batch_size) and (self._last_batch_size != batch_size):
             self.reset_states(batch_size)
 
+        if not len(self._context):
+            self._context = torch.zeros(batch_size, context_size)
+
+        x = torch.cat([self._context, x], dim=1)
         if sr in [8000, 16000]:
-            ort_inputs = {'input': x.numpy(), 'h': self._h, 'c': self._c, 'sr': np.array(sr, dtype='int64')}
+            ort_inputs = {'input': x.numpy(), 'state': self._state.numpy(), 'sr': np.array(sr, dtype='int64')}
             ort_outs = self.session.run(None, ort_inputs)
-            out, self._h, self._c = ort_outs
+            out, state = ort_outs
+            self._state = torch.from_numpy(state)
         else:
             raise ValueError()
 
+        self._context = x[..., -context_size:]
         self._last_sr = sr
         self._last_batch_size = batch_size
 
-        out = torch.tensor(out)
+        out = torch.from_numpy(out)
         return out
 
-    def audio_forward(self, x, sr: int, num_samples: int = 512):
+    def audio_forward(self, x, sr: int):
         outs = []
         x, sr = self._validate_input(x, sr)
+        self.reset_states()
+        num_samples = 512 if sr == 16000 else 256
 
         if x.shape[1] % num_samples:
             pad_num = num_samples - (x.shape[1] % num_samples)
             x = torch.nn.functional.pad(x, (0, pad_num), 'constant', value=0.0)
 
-        self.reset_states(x.shape[0])
         for i in range(0, x.shape[1], num_samples):
             wavs_batch = x[:, i:i+num_samples]
             out_chunk = self.__call__(wavs_batch, sr)
@@ -94,7 +109,7 @@ class VoiceActivityDetection():
         return stacked.cpu()
 
     @staticmethod
-    def download(model_url="https://github.com/snakers4/silero-vad/raw/master/files/silero_vad.onnx"):
+    def download(model_url="https://github.com/snakers4/silero-vad/raw/v5.0/files/silero_vad.onnx"):
         target_dir = os.path.expanduser("~/.cache/whisper-live/")
 
         # Ensure the target directory exists
@@ -138,5 +153,5 @@ class VoiceActivityDetector:
             bool: True if the speech probability exceeds the threshold, indicating the presence of voice activity;
                   False otherwise.
         """
-        speech_prob = self.model(torch.from_numpy(audio_frame), self.frame_rate).item()
-        return speech_prob > self.threshold
+        speech_probs = self.model.audio_forward(torch.from_numpy(audio_frame.copy()), self.frame_rate)[0]
+        return torch.any(speech_probs > self.threshold).item()