diff --git a/Audio-Transcription-Chrome/background.js b/Audio-Transcription-Chrome/background.js
index 99848d7..8f51b67 100644
--- a/Audio-Transcription-Chrome/background.js
+++ b/Audio-Transcription-Chrome/background.js
@@ -190,17 +190,19 @@ async function stopCapture() {
* Listens for messages from the runtime and performs corresponding actions.
* @param {Object} message - The message received from the runtime.
*/
-chrome.runtime.onMessage.addListener((message) => {
+chrome.runtime.onMessage.addListener(async (message) => {
if (message.action === "startCapture") {
startCapture(message);
} else if (message.action === "stopCapture") {
stopCapture();
} else if (message.action === "updateSelectedLanguage") {
- console.log("Selected language");
- console.log(message.detectedLanguage);
const detectedLanguage = message.detectedLanguage;
chrome.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage });
chrome.storage.local.set({ selectedLanguage: detectedLanguage });
+ } else if (message.action === "toggleCaptureButtons") {
+ chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false });
+ chrome.storage.local.set({ capturingState: { isCapturing: false } })
+ stopCapture();
}
});
diff --git a/Audio-Transcription-Chrome/content.js b/Audio-Transcription-Chrome/content.js
index e8c654f..d1f2c6c 100644
--- a/Audio-Transcription-Chrome/content.js
+++ b/Audio-Transcription-Chrome/content.js
@@ -6,6 +6,52 @@ var elem_text = null;
var segments = [];
var text_segments = [];
+function initPopupElement() {
+ if (document.getElementById('popupElement')) {
+ return;
+ }
+
+ const popupContainer = document.createElement('div');
+ popupContainer.id = 'popupElement';
+ popupContainer.style.cssText = 'position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%); background: white; color: black; padding: 16px; border-radius: 10px; box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.5); display: none; text-align: center;';
+
+ const popupText = document.createElement('span');
+ popupText.textContent = 'Default Text';
+ popupText.className = 'popupText';
+ popupText.style.fontSize = '24px';
+ popupContainer.appendChild(popupText);
+
+ const buttonContainer = document.createElement('div');
+ buttonContainer.style.marginTop = '8px';
+ const closePopupButton = document.createElement('button');
+ closePopupButton.textContent = 'Close';
+ closePopupButton.style.backgroundColor = '#65428A';
+ closePopupButton.style.color = 'white';
+ closePopupButton.style.border = 'none';
+ closePopupButton.style.padding = '8px 16px'; // Add padding for better click area
+ closePopupButton.style.cursor = 'pointer';
+ closePopupButton.addEventListener('click', async () => {
+ popupContainer.style.display = 'none';
+ await browser.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false });
+ });
+ buttonContainer.appendChild(closePopupButton);
+ popupContainer.appendChild(buttonContainer);
+
+ document.body.appendChild(popupContainer);
+}
+
+
+function showPopup(customText) {
+ const popup = document.getElementById('popupElement');
+ const popupText = popup.querySelector('.popupText');
+
+ if (popup && popupText) {
+ popupText.textContent = customText || 'Default Text'; // Set default text if custom text is not provided
+ popup.style.display = 'block';
+ }
+}
+
+
function init_element() {
if (document.getElementById('transcription')) {
return;
@@ -128,11 +174,18 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => {
remove_element();
sendResponse({data: "STOPPED"});
return;
+ } else if (type === "showWaitPopup"){
+ initPopupElement();
+
+ showPopup(`Estimated wait time ~ ${Math.round(data)} minutes`);
+ sendResponse({data: "popup"});
+ return;
}
init_element();
message = JSON.parse(data);
+ message = message["segments"];
var text = '';
for (var i = 0; i < message.length; i++) {
diff --git a/Audio-Transcription-Chrome/options.html b/Audio-Transcription-Chrome/options.html
index 51ab923..a26367f 100644
--- a/Audio-Transcription-Chrome/options.html
+++ b/Audio-Transcription-Chrome/options.html
@@ -10,7 +10,6 @@
-
diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js
index 9f685b7..756fe18 100644
--- a/Audio-Transcription-Chrome/options.js
+++ b/Audio-Transcription-Chrome/options.js
@@ -66,6 +66,16 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) {
return resampledData;
}
+function generateUUID() {
+ let dt = new Date().getTime();
+ const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
+ const r = (dt + Math.random() * 16) % 16 | 0;
+ dt = Math.floor(dt / 16);
+ return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
+ });
+ return uuid;
+}
+
/**
* Starts recording audio from the captured tab.
@@ -73,6 +83,7 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) {
*/
async function startRecord(option) {
const stream = await captureTabAudio();
+ const uuid = generateUUID();
if (stream) {
// call when the stream inactive
@@ -88,6 +99,7 @@ async function startRecord(option) {
socket.onopen = function(e) {
socket.send(
JSON.stringify({
+ uid: uuid,
multilingual: option.multilingual,
language: option.language,
task: option.task
@@ -96,13 +108,26 @@ async function startRecord(option) {
};
socket.onmessage = async (event) => {
+ const data = JSON.parse(event.data);
+ if (data["uid"] !== uuid)
+ return;
+
+ if (data["status"] === "WAIT"){
+ await sendMessageToTab(option.currentTabId, {
+ type: "showWaitPopup",
+ data: data["message"],
+ });
+ chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false })
+ chrome.runtime.sendMessage({ action: "stopCapture" })
+ return;
+ }
+
if (isServerReady === false){
isServerReady = true;
return;
}
if (language === null) {
- const data = JSON.parse(event.data);
language = data["language"];
// send message to popup.js to update dropdown
@@ -115,6 +140,11 @@ async function startRecord(option) {
return;
}
+ if (data["message"] === "DISCONNECT"){
+ chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false })
+ return;
+ }
+
res = await sendMessageToTab(option.currentTabId, {
type: "transcript",
data: event.data,
@@ -135,7 +165,6 @@ async function startRecord(option) {
audioDataCache.push(inputData);
- // feed inputs and run
socket.send(audioData16kHz);
};
diff --git a/Audio-Transcription-Chrome/popup.js b/Audio-Transcription-Chrome/popup.js
index ac05d6c..f67f8b7 100644
--- a/Audio-Transcription-Chrome/popup.js
+++ b/Audio-Transcription-Chrome/popup.js
@@ -119,6 +119,8 @@ document.addEventListener("DOMContentLoaded", function () {
startButton.disabled = isCapturing;
stopButton.disabled = !isCapturing;
useServerCheckbox.disabled = isCapturing;
+ useMultilingualCheckbox.disabled = isCapturing;
+
startButton.classList.toggle("disabled", isCapturing);
stopButton.classList.toggle("disabled", !isCapturing);
}
@@ -165,5 +167,12 @@ document.addEventListener("DOMContentLoaded", function () {
}
}
});
+
+ chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
+ if (request.action === "toggleCaptureButtons") {
+ toggleCaptureButtons(false);
+ chrome.storage.local.set({ capturingState: { isCapturing: false } })
+ }
+ });
});
diff --git a/Audio-Transcription-Firefox/background.js b/Audio-Transcription-Firefox/background.js
index 4c57a84..37130d4 100644
--- a/Audio-Transcription-Firefox/background.js
+++ b/Audio-Transcription-Firefox/background.js
@@ -1,7 +1,7 @@
-browser.runtime.onMessage.addListener(function(request, sender, sendResponse) {
+browser.runtime.onMessage.addListener(async function(request, sender, sendResponse) {
const { action, data } = request;
if (action === "transcript") {
- browser.tabs.query({ active: true, currentWindow: true })
+ await browser.tabs.query({ active: true, currentWindow: true })
.then((tabs) => {
const tabId = tabs[0].id;
browser.tabs.sendMessage(tabId, { action: "show_transcript", data });
@@ -12,9 +12,53 @@ browser.runtime.onMessage.addListener(function(request, sender, sendResponse) {
}
if (action === "updateSelectedLanguage") {
const detectedLanguage = data;
- if (detectedLanguage) {
- browser.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage });
- browser.storage.local.set({ selectedLanguage: detectedLanguage });
+ try {
+ await browser.storage.local.set({ selectedLanguage: detectedLanguage });
+ browser.tabs.query({ active: true, currentWindow: true }).then((tabs) => {
+ const tabId = tabs[0].id;
+ browser.tabs.sendMessage(tabId, { action: "updateSelectedLanguage", detectedLanguage });
+ });
+ } catch (error) {
+ console.error("Error updateSelectedLanguage:", error);
+ }
+ }
+ if (action === "toggleCaptureButtons") {
+ try {
+ await browser.storage.local.set({ capturingState: { isCapturing: false } });
+ browser.tabs.query({ active: true, currentWindow: true }).then((tabs) => {
+ const tabId = tabs[0].id;
+ browser.tabs.sendMessage(tabId, { action: "toggleCaptureButtons", data: false });
+ });
+ } catch (error) {
+ console.error("Error updating capturing state:", error);
+ }
+
+ try{
+ await browser.tabs.query({ active: true, currentWindow: true })
+ .then((tabs) => {
+ const tabId = tabs[0].id;
+ browser.tabs.sendMessage(tabId, { action: "stopCapture", data });
+ })
+ .catch((error) => {
+ console.error("Error retrieving active tab:", error);
+ });
+ } catch (error) {
+ console.error(error);
+ }
+ }
+
+ if (action === "showPopup") {
+ try{
+ await browser.tabs.query({ active: true, currentWindow: true })
+ .then((tabs) => {
+ const tabId = tabs[0].id;
+ browser.tabs.sendMessage(tabId, { action: "showWaitPopup", data });
+ })
+ .catch((error) => {
+ console.error(error);
+ });
+ } catch (error) {
+ console.error(error);
}
}
});
diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js
index d849b0d..1d866c8 100644
--- a/Audio-Transcription-Firefox/content.js
+++ b/Audio-Transcription-Firefox/content.js
@@ -5,6 +5,30 @@ let audioContext = null;
let scriptProcessor = null;
let language = null;
+let isPaused = false;
+
+const mediaElements = document.querySelectorAll('video, audio');
+mediaElements.forEach((mediaElement) => {
+ mediaElement.addEventListener('play', handlePlaybackStateChange);
+ mediaElement.addEventListener('pause', handlePlaybackStateChange);
+});
+
+
+function handlePlaybackStateChange(event) {
+ isPaused = event.target.paused;
+}
+
+function generateUUID() {
+ let dt = new Date().getTime();
+ const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
+ const r = (dt + Math.random() * 16) % 16 | 0;
+ dt = Math.floor(dt / 16);
+ return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
+ });
+ return uuid;
+}
+
+
/**
* Resamples the audio data to a target sample rate of 16kHz.
* @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data.
@@ -45,9 +69,12 @@ function startRecording(data) {
if (language === null && !data.useMultilingual) {
language = 'en';
}
+
+ const uuid = generateUUID();
socket.onopen = function(e) {
socket.send(
JSON.stringify({
+ uid: uuid,
multilingual: data.useMultilingual,
language: data.language,
task: data.task
@@ -56,25 +83,33 @@ function startRecording(data) {
};
let isServerReady = false;
- socket.onmessage = (event) => {
- if (!isServerReady){
+ socket.onmessage = async (event) => {
+ const data = JSON.parse(event.data);
+ if (data["uid"] !== uuid)
+ return;
+
+ if (data["status"] === "WAIT"){
+ await browser.runtime.sendMessage({ action: "showPopup", data: data["message"] })
+ return;
+ }
+
+ if (!isServerReady && data["message"] === "SERVER_READY"){
isServerReady = true;
return;
}
if (language === null ){
- const data = JSON.parse(event.data);
language = data["language"];
-
- browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language })
- .catch(function(error) {
- console.error("Error sending message:", error);
- });
+ await browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language })
return
}
- const data = event.data;;
- browser.runtime.sendMessage({ action: "transcript", data })
+ if (data["message"] === "DISCONNECT"){
+ await browser.runtime.sendMessage({ action: "toggleCaptureButtons", data: false })
+ return
+ }
+
+ await browser.runtime.sendMessage({ action: "transcript", data: event.data })
.catch(function(error) {
console.error("Error sending message:", error);
});
@@ -90,14 +125,13 @@ function startRecording(data) {
recorder = audioContext.createScriptProcessor(4096, 1, 1);
recorder.onaudioprocess = async (event) => {
- if (!audioContext || !isCapturing || !isServerReady) return;
+ if (!audioContext || !isCapturing || !isServerReady || isPaused) return;
const inputData = event.inputBuffer.getChannelData(0);
const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate);
audioDataCache.push(inputData);
-
- // feed inputs and run
+
socket.send(audioData16kHz);
};
@@ -113,6 +147,52 @@ var elem_text = null;
var segments = [];
var text_segments = [];
+function initPopupElement() {
+ if (document.getElementById('popupElement')) {
+ return;
+ }
+
+ const popupContainer = document.createElement('div');
+ popupContainer.id = 'popupElement';
+ popupContainer.style.cssText = 'position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%); background: white; color: black; padding: 16px; border-radius: 10px; box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.5); display: none; text-align: center;';
+
+ const popupText = document.createElement('span');
+ popupText.textContent = 'Default Text';
+ popupText.className = 'popupText';
+ popupText.style.fontSize = '24px';
+ popupContainer.appendChild(popupText);
+
+ const buttonContainer = document.createElement('div');
+ buttonContainer.style.marginTop = '8px';
+ const closePopupButton = document.createElement('button');
+ closePopupButton.textContent = 'Close';
+ closePopupButton.style.backgroundColor = '#65428A';
+ closePopupButton.style.color = 'white';
+ closePopupButton.style.border = 'none';
+ closePopupButton.style.padding = '8px 16px'; // Add padding for better click area
+ closePopupButton.style.cursor = 'pointer';
+ closePopupButton.addEventListener('click', async () => {
+ popupContainer.style.display = 'none';
+ await browser.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false });
+ });
+ buttonContainer.appendChild(closePopupButton);
+ popupContainer.appendChild(buttonContainer);
+
+ document.body.appendChild(popupContainer);
+}
+
+
+function showPopup(customText) {
+ const popup = document.getElementById('popupElement');
+ const popupText = popup.querySelector('.popupText');
+
+ if (popup && popupText) {
+ popupText.textContent = customText || 'Default Text'; // Set default text if custom text is not provided
+ popup.style.display = 'block';
+ }
+}
+
+
function init_element() {
if (document.getElementById('transcription')) {
return;
@@ -250,10 +330,17 @@ browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
remove_element();
+ } else if (action === "showWaitPopup") {
+
+ initPopupElement();
+
+ showPopup(`Estimated wait time ~ ${Math.round(data)} minutes`);
+
} else if (action === "show_transcript"){
if (!isCapturing) return;
init_element();
message = JSON.parse(data);
+ message = message["segments"];
var text = '';
for (var i = 0; i < message.length; i++) {
diff --git a/Audio-Transcription-Firefox/popup.html b/Audio-Transcription-Firefox/popup.html
index 7393898..7ca020a 100644
--- a/Audio-Transcription-Firefox/popup.html
+++ b/Audio-Transcription-Firefox/popup.html
@@ -15,6 +15,8 @@
+
+
diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js
index 70923c3..a707fca 100644
--- a/Audio-Transcription-Firefox/popup.js
+++ b/Audio-Transcription-Firefox/popup.js
@@ -113,7 +113,9 @@ document.addEventListener("DOMContentLoaded", function() {
function toggleCaptureButtons(isCapturing) {
startButton.disabled = isCapturing;
stopButton.disabled = !isCapturing;
- useServerCheckbox.disabled = isCapturing; // Disable checkbox if capturing
+ useServerCheckbox.disabled = isCapturing;
+ useMultilingualCheckbox.disabled = isCapturing;
+
startButton.classList.toggle("disabled", isCapturing);
stopButton.classList.toggle("disabled", !isCapturing);
}
@@ -152,7 +154,7 @@ document.addEventListener("DOMContentLoaded", function() {
browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
if (request.action === "updateSelectedLanguage") {
- const detectedLanguage = request.detectedLanguage;
+ const detectedLanguage = request.data;
if (detectedLanguage) {
languageDropdown.value = detectedLanguage;
@@ -161,4 +163,14 @@ document.addEventListener("DOMContentLoaded", function() {
}
}
});
+
+ browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
+ if (request.action === "toggleCaptureButtons") {
+ toggleCaptureButtons(false);
+ browser.storage.local.set({ capturingState: { isCapturing: false } })
+ .catch(function(error) {
+ console.error("Error storing capturing state:", error);
+ });
+ }
+ });
});
diff --git a/run_server.py b/run_server.py
index 728e22b..cac0f93 100644
--- a/run_server.py
+++ b/run_server.py
@@ -1,6 +1,5 @@
-
from whisper_live.server import TranscriptionServer
if __name__ == "__main__":
server = TranscriptionServer()
- server.run("0.0.0.0", 9090)
+ server.run("0.0.0.0")
diff --git a/setup.py b/setup.py
index 64b99fb..1b23d24 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@ README = (HERE / "README.md").read_text()
# This call to setup() does all the work
setup(name="whisper-live",
- version="0.0.5",
+ version="0.0.6",
description="A nearly-live implementation of OpenAI's Whisper.",
long_description=README,
long_description_content_type="text/markdown",
diff --git a/whisper_live/client.py b/whisper_live/client.py
index 158e364..e397c18 100644
--- a/whisper_live/client.py
+++ b/whisper_live/client.py
@@ -10,6 +10,7 @@ import threading
import textwrap
import json
import websocket
+import uuid
def resample(file: str, sr: int = 16000):
@@ -47,10 +48,12 @@ class Client:
CHANNELS = 1
RATE = 16000
RECORD_SECONDS = 60000
- START_RECORDING = False
+ RECORDING = False
multilingual = False
language = None
task = "transcribe"
+ uid = str(uuid.uuid4())
+ WAITING = False
def __init__(self, host=None, port=None, is_multilingual=False, lang=None, translate=False):
Client.multilingual = is_multilingual
@@ -90,16 +93,32 @@ class Client:
@staticmethod
def on_message(ws, message):
message = json.loads(message)
- if message == "SERVER_READY":
- Client.START_RECORDING = True
+ if message.get('uid')!=Client.uid:
+ print("[ERROR]: invalid client uid")
+ return
+
+ if "status" in message.keys() and message["status"] == "WAIT":
+ Client.WAITING = True
+ print(f"[INFO]:Server is full. Estimated wait time {round(message['message'])} minutes.")
+
+ if "message" in message.keys() and message["message"] == "DISCONNECT":
+ print("[INFO]: Server overtime disconnected.")
+ Client.RECORDING = False
+
+ if "message" in message.keys() and message["message"] == "SERVER_READY":
+ Client.RECORDING = True
return
- if isinstance(message, dict):
+ if "language" in message.keys():
Client.language = message.get("language")
lang_prob = message.get("language_prob")
print(f"[INFO]: Server detected language {Client.language} with probability {lang_prob}")
return
+ if "segments" not in message.keys():
+ return
+
+ message = message["segments"]
text = []
if len(message):
for seg in message:
@@ -134,6 +153,7 @@ class Client:
print("[INFO]: Opened connection")
ws.send(json.dumps({
+ 'uid': Client.uid,
'multilingual': Client.multilingual,
'language': Client.language,
'task': Client.task
@@ -162,7 +182,7 @@ class Client:
output=True,
frames_per_buffer=self.CHUNK)
try:
- while True:
+ while Client.RECORDING:
data = self.wf.readframes(self.CHUNK)
if data==b'': break
@@ -210,6 +230,7 @@ class Client:
os.makedirs("chunks", exist_ok=True)
try:
for _ in range(0, int(self.RATE / self.CHUNK * self.RECORD_SECONDS)):
+ if not Client.RECORDING: break
data = self.stream.read(self.CHUNK)
self.frames += data
@@ -264,7 +285,10 @@ class TranscriptionClient:
def __call__(self, audio=None):
print("[INFO]: Waiting for server ready ...")
- while not Client.START_RECORDING:
+ while not Client.RECORDING:
+ if Client.WAITING:
+ self.client.close_websocket()
+ return
pass
print("[INFO]: Server Ready!")
if audio is not None:
diff --git a/whisper_live/server.py b/whisper_live/server.py
index cb2d51c..aa4439a 100644
--- a/whisper_live/server.py
+++ b/whisper_live/server.py
@@ -2,11 +2,12 @@ import websockets
import pickle, struct, time, pyaudio
import threading
import os, json
+import base64
import wave
import textwrap
import logging
-logging.basicConfig(level = logging.INFO)
+# logging.basicConfig(level = logging.INFO)
from collections import deque
from dataclasses import dataclass
@@ -14,6 +15,7 @@ from websockets.sync.server import serve
import torch
import numpy as np
+import time
from whisper_live.transcriber import WhisperModel
@@ -24,9 +26,30 @@ class TranscriptionServer:
Attributes:
clients (dict): A dictionary to store connected clients.
"""
-
+ RATE = 16000
def __init__(self):
+ # voice activity detection model
+ self.vad_model, _ = torch.hub.load(repo_or_dir='snakers4/silero-vad',
+ model='silero_vad',
+ force_reload=True,
+ onnx=True
+ )
+ self.vad_threshold = 0.4
self.clients = {}
+ self.websockets = {}
+ self.clients_start_time = {}
+ self.max_clients = 4
+ self.max_connection_time = 600 # in seconds
+
+ def get_wait_time(self):
+ wait_time = None
+ for k,v in self.clients_start_time.items():
+ current_client_time_remaining = self.max_connection_time - (time.time() - v)
+ if wait_time is None:
+ wait_time = current_client_time_remaining
+ elif current_client_time_remaining < wait_time:
+ wait_time = current_client_time_remaining
+ return wait_time/60
def recv_audio(self, websocket):
"""
@@ -35,30 +58,73 @@ class TranscriptionServer:
Args:
websocket (WebSocket): The WebSocket connection for the client.
"""
+ logging.info("New client connected")
options = websocket.recv()
options = json.loads(options)
+
+ if len(self.clients) >= self.max_clients:
+ logging.warning("Client Queue Full. Asking client to wait ...")
+ wait_time = self.get_wait_time()
+ response = {
+ "uid" : options["uid"],
+ "status": "WAIT",
+ "message": wait_time,
+ }
+ websocket.send(json.dumps(response))
+ websocket.close()
+ del websocket
+ return
+
client = ServeClient(
websocket,
multilingual=options["multilingual"],
language=options["language"],
task=options["task"],
+ client_uid=options["uid"]
)
-
+
self.clients[websocket] = client
+ self.clients_start_time[websocket] = time.time()
while True:
try:
frame_data = websocket.recv()
- frame_np = np.frombuffer(frame_data, np.float32)
+ frame_np = np.frombuffer(frame_data, dtype=np.float32)
+
+ try:
+ speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item()
+ if speech_prob < self.vad_threshold:
+ continue
+
+ except Exception as e:
+ logging.error(e)
+ return
self.clients[websocket].add_frames(frame_np)
+ elapsed_time = time.time() - self.clients_start_time[websocket]
+ if elapsed_time >= self.max_connection_time:
+ self.clients[websocket].disconnect()
+ logging.warning(f"{self.clients[websocket]} Client disconnected due to overtime.")
+ self.clients[websocket].cleanup()
+ self.clients.pop(websocket)
+ self.clients_start_time.pop(websocket)
+ websocket.close()
+ del websocket
+ break
+
+
except Exception as e:
+ logging.error(e)
self.clients[websocket].cleanup()
self.clients.pop(websocket)
+ self.clients_start_time.pop(websocket)
logging.info("Connection Closed.")
+ logging.info(self.clients)
+
+ del websocket
break
- def run(self, host, port):
+ def run(self, host, port=9090):
"""
Run the transcription server.
@@ -73,8 +139,10 @@ class TranscriptionServer:
class ServeClient:
RATE = 16000
SERVER_READY = "SERVER_READY"
+ DISCONNECT = "DISCONNECT"
- def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None):
+ def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None, client_uid=None):
+ self.client_uid = client_uid
self.data = b""
self.frames = b""
self.language = language if multilingual else "en"
@@ -87,14 +155,6 @@ class ServeClient:
local_files_only=False,
)
- # voice activity detection model
- self.vad_model, _ = torch.hub.load(repo_or_dir='snakers4/silero-vad',
- model='silero_vad',
- force_reload=True,
- onnx=True
- )
- self.vad_threshold = 0.4
-
self.timestamp_offset = 0.0
self.frames_np = None
self.frames_offset = 0.0
@@ -117,7 +177,14 @@ class ServeClient:
self.websocket = websocket
self.trans_thread = threading.Thread(target=self.speech_to_text)
self.trans_thread.start()
- self.websocket.send(json.dumps(self.SERVER_READY))
+ self.websocket.send(
+ json.dumps(
+ {
+ "uid": self.client_uid,
+ "message": self.SERVER_READY
+ }
+ )
+ )
def fill_output(self, output):
"""
@@ -142,15 +209,6 @@ class ServeClient:
return wrapped
def add_frames(self, frame_np):
- try:
- speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item()
- if speech_prob < self.vad_threshold:
- return
-
- except Exception as e:
- logging.error(e)
- return
-
if self.frames_np is not None and self.frames_np.shape[0] > 45*self.RATE:
self.frames_offset += 30.0
self.frames_np = self.frames_np[int(30*self.RATE):]
@@ -179,7 +237,8 @@ class ServeClient:
task=self.task
)
logging.info(f"Detected language {self.language} with probability {lang_prob}")
- self.websocket.send(json.dumps({"language": self.language, "language_prob": lang_prob}))
+ self.websocket.send(json.dumps(
+ {"uid": self.client_uid, "language": self.language, "language_prob": lang_prob}))
while True:
if self.exit:
@@ -227,9 +286,14 @@ class ServeClient:
segments = segments + [last_segment]
try:
- self.websocket.send(json.dumps(segments))
+ self.websocket.send(
+ json.dumps({
+ "uid": self.client_uid,
+ "segments": segments
+ })
+ )
except Exception as e:
- logging.info(f"[ERROR]: {e}")
+ logging.error(f"[ERROR]: {e}")
else:
# show previous output if there is pause i.e. no output from whisper
segments = []
@@ -246,11 +310,16 @@ class ServeClient:
self.text.append('')
try:
- self.websocket.send(json.dumps(segments))
+ self.websocket.send(
+ json.dumps({
+ "uid": self.client_uid,
+ "segments": segments
+ })
+ )
except Exception as e:
- logging.info(f"[INFO]: {e}")
+ logging.error(f"[ERROR]: {e}")
except Exception as e:
- logging.info(f"[INFO]: {e}")
+ logging.error(f"[ERROR]: {e}")
time.sleep(0.01)
def update_segments(self, segments, duration):
@@ -321,8 +390,17 @@ class ServeClient:
return last_segment
+ def disconnect(self):
+ self.websocket.send(
+ json.dumps(
+ {
+ "uid": self.client_uid,
+ "message": self.DISCONNECT
+ }
+ )
+ )
+
def cleanup(self):
logging.info("Cleaning up.")
self.exit = True
self.transcriber.destroy()
-