From 1a4775bac2b02ec8a0df1376e62f210855ea7e88 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 7 Aug 2023 22:07:51 +0800 Subject: [PATCH 01/13] add client queue --- run_server.py | 3 +- whisper_live/server.py | 126 +++++++++++++++++++++++++++++++---------- 2 files changed, 99 insertions(+), 30 deletions(-) diff --git a/run_server.py b/run_server.py index 728e22b..aff2cd1 100644 --- a/run_server.py +++ b/run_server.py @@ -1,6 +1,7 @@ +import asyncio from whisper_live.server import TranscriptionServer if __name__ == "__main__": server = TranscriptionServer() - server.run("0.0.0.0", 9090) + server.run("0.0.0.0") diff --git a/whisper_live/server.py b/whisper_live/server.py index f368bd2..100e955 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -2,11 +2,12 @@ import websockets import pickle, struct, time, pyaudio import threading import os, json +import base64 import wave import textwrap import logging -logging.basicConfig(level = logging.INFO) +# logging.basicConfig(level = logging.INFO) from collections import deque from dataclasses import dataclass @@ -14,6 +15,7 @@ from websockets.sync.server import serve import torch import numpy as np +import time from whisper_live.transcriber import WhisperModel @@ -24,9 +26,19 @@ class TranscriptionServer: Attributes: clients (dict): A dictionary to store connected clients. """ - + RATE = 16000 def __init__(self): + # voice activity detection model + self.vad_model, _ = torch.hub.load(repo_or_dir='snakers4/silero-vad', + model='silero_vad', + force_reload=True, + onnx=True + ) + self.vad_threshold = 0.4 self.clients = {} + self.websockets = {} + self.clients_start_time = {} + self.max_clients = 4 def recv_audio(self, websocket): """ @@ -35,6 +47,18 @@ class TranscriptionServer: Args: websocket (WebSocket): The WebSocket connection for the client. """ + # Check if the maximum number of clients is reached + print("New client connected") + if len(self.clients) >= self.max_clients: + # Send response to the new client to come back later + response = { + "status": "error", + "message": "Server is currently full. Please try again later.", + } + websocket.send(json.dumps(response)) + websocket.close() + return + options = websocket.recv() options = json.loads(options) client = ServeClient( @@ -42,23 +66,55 @@ class TranscriptionServer: multilingual=options["multilingual"], language=options["language"], task=options["task"], + client_uid=options["uid"] ) - + self.clients[websocket] = client + # max 10 minutes for each client + self.clients_start_time[websocket] = time.time() while True: try: frame_data = websocket.recv() - frame_np = np.frombuffer(frame_data, np.float32) + data = json.loads(frame_data) + base64_audio = data["audio"] + binary_audio = base64.b64decode(base64_audio) + frame_np = np.frombuffer(binary_audio, dtype=np.float32) + + try: + speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item() + if speech_prob < self.vad_threshold: + continue + + except Exception as e: + logging.error(e) + return self.clients[websocket].add_frames(frame_np) + elapsed_time = time.time() - self.clients_start_time[websocket] + if elapsed_time >= 45: # 10 minutes in seconds + # send a disconnection message + self.clients[websocket].disconnect() + print(f"{self.clients[websocket]} Client disconnected due to overtime.") + print() + self.clients[websocket].cleanup() + self.clients.pop(websocket) + self.clients_start_time.pop(websocket) + websocket.close() + del websocket + break + except Exception as e: self.clients[websocket].cleanup() self.clients.pop(websocket) - logging.info("Connection Closed.") + self.clients_start_time.pop(websocket) + print("Connection Closed.") + print(self.clients) + + del websocket break - def run(self, host, port): + def run(self, host, port=9090): """ Run the transcription server. @@ -73,8 +129,10 @@ class TranscriptionServer: class ServeClient: RATE = 16000 SERVER_READY = "SERVER_READY" + DISCONNECT = "DISCONNECT" - def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None): + def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None, client_uid=None): + self.client_uid = client_uid self.data = b"" self.frames = b"" self.language = language if multilingual else "en" @@ -86,14 +144,6 @@ class ServeClient: local_files_only=False, ) - # voice activity detection model - self.vad_model, _ = torch.hub.load(repo_or_dir='snakers4/silero-vad', - model='silero_vad', - force_reload=True, - onnx=True - ) - self.vad_threshold = 0.4 - self.timestamp_offset = 0.0 self.frames_np = None self.frames_offset = 0.0 @@ -116,7 +166,14 @@ class ServeClient: self.websocket = websocket self.trans_thread = threading.Thread(target=self.speech_to_text) self.trans_thread.start() - self.websocket.send(json.dumps(self.SERVER_READY)) + self.websocket.send( + json.dumps( + { + "uid": self.client_uid, + "message": self.SERVER_READY + } + ) + ) def fill_output(self, output): """ @@ -141,15 +198,6 @@ class ServeClient: return wrapped def add_frames(self, frame_np): - try: - speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item() - if speech_prob < self.vad_threshold: - return - - except Exception as e: - logging.error(e) - return - if self.frames_np is not None and self.frames_np.shape[0] > 45*self.RATE: self.frames_offset += 30.0 self.frames_np = self.frames_np[int(30*self.RATE):] @@ -178,7 +226,8 @@ class ServeClient: task=self.task ) logging.info(f"Detected language {self.language} with probability {lang_prob}") - self.websocket.send(json.dumps({"language": self.language, "language_prob": lang_prob})) + self.websocket.send(json.dumps( + {"uid": self.client_uid, "language": self.language, "language_prob": lang_prob})) while True: if self.exit: @@ -226,7 +275,12 @@ class ServeClient: segments = segments + [last_segment] try: - self.websocket.send(json.dumps(segments)) + self.websocket.send( + json.dumps({ + "uid": self.client_uid, + "segments": segments + }) + ) except Exception as e: logging.info(f"[ERROR]: {e}") else: @@ -245,7 +299,12 @@ class ServeClient: self.text.append('') try: - self.websocket.send(json.dumps(segments)) + self.websocket.send( + json.dumps({ + "uid": self.client_uid, + "segments": segments + }) + ) except Exception as e: logging.info(f"[INFO]: {e}") except Exception as e: @@ -320,8 +379,17 @@ class ServeClient: return last_segment + def disconnect(self): + self.websocket.send( + json.dumps( + { + "uid": self.client_uid, + "message": self.DISCONNECT + } + ) + ) + def cleanup(self): logging.info("Cleaning up.") self.exit = True self.transcriber.destroy() - From d2336c6561e531e09615b023e0e114e8e59c39e8 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 14:29:30 +0800 Subject: [PATCH 02/13] return wait time to client --- whisper_live/server.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/whisper_live/server.py b/whisper_live/server.py index 100e955..f4690f4 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -39,6 +39,17 @@ class TranscriptionServer: self.websockets = {} self.clients_start_time = {} self.max_clients = 4 + self.max_connection_time = 600 # in seconds + + def get_wait_time(self): + wait_time = None + for k,v in self.clients_start_time.items(): + current_client_time_remaining = self.max_connection_time - (time.time() - v) + if wait_time is None: + wait_time = current_client_time_remaining + elif current_client_time_remaining < wait_time: + wait_time = current_client_time_remaining + return wait_time/60 def recv_audio(self, websocket): """ @@ -51,9 +62,10 @@ class TranscriptionServer: print("New client connected") if len(self.clients) >= self.max_clients: # Send response to the new client to come back later + wait_time = self.get_wait_time() response = { - "status": "error", - "message": "Server is currently full. Please try again later.", + "status": "WAIT", + "message": wait_time, } websocket.send(json.dumps(response)) websocket.close() From 19b9d8b03d9ef47438ff8c40d010b4e7b226b2e8 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 17:33:29 +0800 Subject: [PATCH 03/13] update server with client queue and connection time --- whisper_live/server.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/whisper_live/server.py b/whisper_live/server.py index f4690f4..14b0fbf 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -38,7 +38,7 @@ class TranscriptionServer: self.clients = {} self.websockets = {} self.clients_start_time = {} - self.max_clients = 4 + self.max_clients = 1 self.max_connection_time = 600 # in seconds def get_wait_time(self): @@ -58,21 +58,23 @@ class TranscriptionServer: Args: websocket (WebSocket): The WebSocket connection for the client. """ - # Check if the maximum number of clients is reached - print("New client connected") + logging.info("New client connected") + options = websocket.recv() + options = json.loads(options) + if len(self.clients) >= self.max_clients: - # Send response to the new client to come back later + logging.warning("Client Queue Full. Asking client to wait ...") wait_time = self.get_wait_time() response = { + "uid" : options["uid"], "status": "WAIT", "message": wait_time, } websocket.send(json.dumps(response)) websocket.close() + del websocket return - options = websocket.recv() - options = json.loads(options) client = ServeClient( websocket, multilingual=options["multilingual"], @@ -82,7 +84,6 @@ class TranscriptionServer: ) self.clients[websocket] = client - # max 10 minutes for each client self.clients_start_time[websocket] = time.time() while True: @@ -102,12 +103,11 @@ class TranscriptionServer: logging.error(e) return self.clients[websocket].add_frames(frame_np) + elapsed_time = time.time() - self.clients_start_time[websocket] - if elapsed_time >= 45: # 10 minutes in seconds - # send a disconnection message + if elapsed_time >= self.max_connection_time: self.clients[websocket].disconnect() - print(f"{self.clients[websocket]} Client disconnected due to overtime.") - print() + logging.warning(f"{self.clients[websocket]} Client disconnected due to overtime.") self.clients[websocket].cleanup() self.clients.pop(websocket) self.clients_start_time.pop(websocket) @@ -120,8 +120,8 @@ class TranscriptionServer: self.clients[websocket].cleanup() self.clients.pop(websocket) self.clients_start_time.pop(websocket) - print("Connection Closed.") - print(self.clients) + logging.info("Connection Closed.") + logging.info(self.clients) del websocket break @@ -294,7 +294,7 @@ class ServeClient: }) ) except Exception as e: - logging.info(f"[ERROR]: {e}") + logging.error(f"[ERROR]: {e}") else: # show previous output if there is pause i.e. no output from whisper segments = [] @@ -318,9 +318,9 @@ class ServeClient: }) ) except Exception as e: - logging.info(f"[INFO]: {e}") + logging.error(f"[ERROR]: {e}") except Exception as e: - logging.info(f"[INFO]: {e}") + logging.error(f"[ERROR]: {e}") time.sleep(0.01) def update_segments(self, segments, duration): From f1df664f06dfd3b1874e075cdfc1356ecbe4cd47 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 15:12:43 +0530 Subject: [PATCH 04/13] skip sending data if content is paused --- Audio-Transcription-Firefox/content.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index d849b0d..53f216f 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -5,6 +5,20 @@ let audioContext = null; let scriptProcessor = null; let language = null; +let isPaused = false; + +const mediaElements = document.querySelectorAll('video, audio'); +mediaElements.forEach((mediaElement) => { + mediaElement.addEventListener('play', handlePlaybackStateChange); + mediaElement.addEventListener('pause', handlePlaybackStateChange); +}); + + +function handlePlaybackStateChange(event) { + isPaused = event.target.paused; +} + + /** * Resamples the audio data to a target sample rate of 16kHz. * @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data. @@ -90,7 +104,7 @@ function startRecording(data) { recorder = audioContext.createScriptProcessor(4096, 1, 1); recorder.onaudioprocess = async (event) => { - if (!audioContext || !isCapturing || !isServerReady) return; + if (!audioContext || !isCapturing || !isServerReady || isPaused) return; const inputData = event.inputBuffer.getChannelData(0); const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate); From 4f88816d275fb70d8a17be1bc6ce81011ced0b97 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 15:14:38 +0530 Subject: [PATCH 05/13] firefox handle server messages: disconnect, wait --- Audio-Transcription-Firefox/background.js | 54 +++++++++-- Audio-Transcription-Firefox/content.js | 104 +++++++++++++++++++--- Audio-Transcription-Firefox/popup.html | 2 + Audio-Transcription-Firefox/popup.js | 16 +++- 4 files changed, 157 insertions(+), 19 deletions(-) diff --git a/Audio-Transcription-Firefox/background.js b/Audio-Transcription-Firefox/background.js index 4c57a84..37130d4 100644 --- a/Audio-Transcription-Firefox/background.js +++ b/Audio-Transcription-Firefox/background.js @@ -1,7 +1,7 @@ -browser.runtime.onMessage.addListener(function(request, sender, sendResponse) { +browser.runtime.onMessage.addListener(async function(request, sender, sendResponse) { const { action, data } = request; if (action === "transcript") { - browser.tabs.query({ active: true, currentWindow: true }) + await browser.tabs.query({ active: true, currentWindow: true }) .then((tabs) => { const tabId = tabs[0].id; browser.tabs.sendMessage(tabId, { action: "show_transcript", data }); @@ -12,9 +12,53 @@ browser.runtime.onMessage.addListener(function(request, sender, sendResponse) { } if (action === "updateSelectedLanguage") { const detectedLanguage = data; - if (detectedLanguage) { - browser.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage }); - browser.storage.local.set({ selectedLanguage: detectedLanguage }); + try { + await browser.storage.local.set({ selectedLanguage: detectedLanguage }); + browser.tabs.query({ active: true, currentWindow: true }).then((tabs) => { + const tabId = tabs[0].id; + browser.tabs.sendMessage(tabId, { action: "updateSelectedLanguage", detectedLanguage }); + }); + } catch (error) { + console.error("Error updateSelectedLanguage:", error); + } + } + if (action === "toggleCaptureButtons") { + try { + await browser.storage.local.set({ capturingState: { isCapturing: false } }); + browser.tabs.query({ active: true, currentWindow: true }).then((tabs) => { + const tabId = tabs[0].id; + browser.tabs.sendMessage(tabId, { action: "toggleCaptureButtons", data: false }); + }); + } catch (error) { + console.error("Error updating capturing state:", error); + } + + try{ + await browser.tabs.query({ active: true, currentWindow: true }) + .then((tabs) => { + const tabId = tabs[0].id; + browser.tabs.sendMessage(tabId, { action: "stopCapture", data }); + }) + .catch((error) => { + console.error("Error retrieving active tab:", error); + }); + } catch (error) { + console.error(error); + } + } + + if (action === "showPopup") { + try{ + await browser.tabs.query({ active: true, currentWindow: true }) + .then((tabs) => { + const tabId = tabs[0].id; + browser.tabs.sendMessage(tabId, { action: "showWaitPopup", data }); + }) + .catch((error) => { + console.error(error); + }); + } catch (error) { + console.error(error); } } }); diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index 53f216f..fae2ac8 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -18,6 +18,16 @@ function handlePlaybackStateChange(event) { isPaused = event.target.paused; } +function generateUUID() { + let dt = new Date().getTime(); + const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = (dt + Math.random() * 16) % 16 | 0; + dt = Math.floor(dt / 16); + return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16); + }); + return uuid; +} + /** * Resamples the audio data to a target sample rate of 16kHz. @@ -59,9 +69,12 @@ function startRecording(data) { if (language === null && !data.useMultilingual) { language = 'en'; } + + const uuid = generateUUID(); socket.onopen = function(e) { socket.send( JSON.stringify({ + uid: uuid, multilingual: data.useMultilingual, language: data.language, task: data.task @@ -70,25 +83,33 @@ function startRecording(data) { }; let isServerReady = false; - socket.onmessage = (event) => { - if (!isServerReady){ + socket.onmessage = async (event) => { + const data = JSON.parse(event.data); + if (data["uid"] !== uuid) + return; + + if (data["status"] === "WAIT"){ + await browser.runtime.sendMessage({ action: "showPopup", data: data["message"] }) + return; + } + + if (!isServerReady && data["message"] === "SERVER_READY"){ isServerReady = true; return; } if (language === null ){ - const data = JSON.parse(event.data); language = data["language"]; - - browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language }) - .catch(function(error) { - console.error("Error sending message:", error); - }); + await browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language }) return } - const data = event.data;; - browser.runtime.sendMessage({ action: "transcript", data }) + if (data["message"] === "DISCONNECT"){ + await browser.runtime.sendMessage({ action: "toggleCaptureButtons", data: false }) + return + } + + await browser.runtime.sendMessage({ action: "transcript", data: event.data }) .catch(function(error) { console.error("Error sending message:", error); }); @@ -110,9 +131,15 @@ function startRecording(data) { const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate); audioDataCache.push(inputData); - + // feed inputs and run - socket.send(audioData16kHz); + const base64AudioData = btoa(String.fromCharCode.apply(null, new Uint8Array(audioData16kHz.buffer))); + + socket.send( + JSON.stringify({ + "audio": base64AudioData + }) + ); }; // Prevent page mute @@ -127,6 +154,52 @@ var elem_text = null; var segments = []; var text_segments = []; +function initPopupElement() { + if (document.getElementById('popupElement')) { + return; + } + + const popupContainer = document.createElement('div'); + popupContainer.id = 'popupElement'; + popupContainer.style.cssText = 'position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%); background: white; color: black; padding: 16px; border-radius: 10px; box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.5); display: none; text-align: center;'; + + const popupText = document.createElement('span'); + popupText.textContent = 'Default Text'; + popupText.className = 'popupText'; + popupText.style.fontSize = '24px'; + popupContainer.appendChild(popupText); + + const buttonContainer = document.createElement('div'); + buttonContainer.style.marginTop = '8px'; + const closePopupButton = document.createElement('button'); + closePopupButton.textContent = 'Close'; + closePopupButton.style.backgroundColor = '#65428A'; + closePopupButton.style.color = 'white'; + closePopupButton.style.border = 'none'; + closePopupButton.style.padding = '8px 16px'; // Add padding for better click area + closePopupButton.style.cursor = 'pointer'; + closePopupButton.addEventListener('click', async () => { + popupContainer.style.display = 'none'; + await browser.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false }); + }); + buttonContainer.appendChild(closePopupButton); + popupContainer.appendChild(buttonContainer); + + document.body.appendChild(popupContainer); +} + + +function showPopup(customText) { + const popup = document.getElementById('popupElement'); + const popupText = popup.querySelector('.popupText'); + + if (popup && popupText) { + popupText.textContent = customText || 'Default Text'; // Set default text if custom text is not provided + popup.style.display = 'block'; + } +} + + function init_element() { if (document.getElementById('transcription')) { return; @@ -264,10 +337,17 @@ browser.runtime.onMessage.addListener((request, sender, sendResponse) => { remove_element(); + } else if (action === "showWaitPopup") { + + initPopupElement(); + + showPopup(`Estimated wait time ~ ${Math.round(data)} minutes`); + } else if (action === "show_transcript"){ if (!isCapturing) return; init_element(); message = JSON.parse(data); + message = message["segments"]; var text = ''; for (var i = 0; i < message.length; i++) { diff --git a/Audio-Transcription-Firefox/popup.html b/Audio-Transcription-Firefox/popup.html index 7393898..7ca020a 100644 --- a/Audio-Transcription-Firefox/popup.html +++ b/Audio-Transcription-Firefox/popup.html @@ -15,6 +15,8 @@ + +
diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js index 70923c3..a707fca 100644 --- a/Audio-Transcription-Firefox/popup.js +++ b/Audio-Transcription-Firefox/popup.js @@ -113,7 +113,9 @@ document.addEventListener("DOMContentLoaded", function() { function toggleCaptureButtons(isCapturing) { startButton.disabled = isCapturing; stopButton.disabled = !isCapturing; - useServerCheckbox.disabled = isCapturing; // Disable checkbox if capturing + useServerCheckbox.disabled = isCapturing; + useMultilingualCheckbox.disabled = isCapturing; + startButton.classList.toggle("disabled", isCapturing); stopButton.classList.toggle("disabled", !isCapturing); } @@ -152,7 +154,7 @@ document.addEventListener("DOMContentLoaded", function() { browser.runtime.onMessage.addListener((request, sender, sendResponse) => { if (request.action === "updateSelectedLanguage") { - const detectedLanguage = request.detectedLanguage; + const detectedLanguage = request.data; if (detectedLanguage) { languageDropdown.value = detectedLanguage; @@ -161,4 +163,14 @@ document.addEventListener("DOMContentLoaded", function() { } } }); + + browser.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.action === "toggleCaptureButtons") { + toggleCaptureButtons(false); + browser.storage.local.set({ capturingState: { isCapturing: false } }) + .catch(function(error) { + console.error("Error storing capturing state:", error); + }); + } + }); }); From b7b4f8a9d2ac19ebf459a3576415c52d8152bc17 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 15:16:14 +0530 Subject: [PATCH 06/13] remove ort script import --- Audio-Transcription-Chrome/options.html | 1 - 1 file changed, 1 deletion(-) diff --git a/Audio-Transcription-Chrome/options.html b/Audio-Transcription-Chrome/options.html index 51ab923..a26367f 100644 --- a/Audio-Transcription-Chrome/options.html +++ b/Audio-Transcription-Chrome/options.html @@ -10,7 +10,6 @@ - From 50e18b5d5f39d292e0eed52d38737dc832728749 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 15:16:55 +0530 Subject: [PATCH 07/13] add popup element and handle server messages --- Audio-Transcription-Chrome/background.js | 8 ++-- Audio-Transcription-Chrome/content.js | 53 ++++++++++++++++++++++++ Audio-Transcription-Chrome/options.js | 40 +++++++++++++++++- Audio-Transcription-Chrome/popup.js | 9 ++++ 4 files changed, 105 insertions(+), 5 deletions(-) diff --git a/Audio-Transcription-Chrome/background.js b/Audio-Transcription-Chrome/background.js index 99848d7..8f51b67 100644 --- a/Audio-Transcription-Chrome/background.js +++ b/Audio-Transcription-Chrome/background.js @@ -190,17 +190,19 @@ async function stopCapture() { * Listens for messages from the runtime and performs corresponding actions. * @param {Object} message - The message received from the runtime. */ -chrome.runtime.onMessage.addListener((message) => { +chrome.runtime.onMessage.addListener(async (message) => { if (message.action === "startCapture") { startCapture(message); } else if (message.action === "stopCapture") { stopCapture(); } else if (message.action === "updateSelectedLanguage") { - console.log("Selected language"); - console.log(message.detectedLanguage); const detectedLanguage = message.detectedLanguage; chrome.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage }); chrome.storage.local.set({ selectedLanguage: detectedLanguage }); + } else if (message.action === "toggleCaptureButtons") { + chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false }); + chrome.storage.local.set({ capturingState: { isCapturing: false } }) + stopCapture(); } }); diff --git a/Audio-Transcription-Chrome/content.js b/Audio-Transcription-Chrome/content.js index e8c654f..d1f2c6c 100644 --- a/Audio-Transcription-Chrome/content.js +++ b/Audio-Transcription-Chrome/content.js @@ -6,6 +6,52 @@ var elem_text = null; var segments = []; var text_segments = []; +function initPopupElement() { + if (document.getElementById('popupElement')) { + return; + } + + const popupContainer = document.createElement('div'); + popupContainer.id = 'popupElement'; + popupContainer.style.cssText = 'position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%); background: white; color: black; padding: 16px; border-radius: 10px; box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.5); display: none; text-align: center;'; + + const popupText = document.createElement('span'); + popupText.textContent = 'Default Text'; + popupText.className = 'popupText'; + popupText.style.fontSize = '24px'; + popupContainer.appendChild(popupText); + + const buttonContainer = document.createElement('div'); + buttonContainer.style.marginTop = '8px'; + const closePopupButton = document.createElement('button'); + closePopupButton.textContent = 'Close'; + closePopupButton.style.backgroundColor = '#65428A'; + closePopupButton.style.color = 'white'; + closePopupButton.style.border = 'none'; + closePopupButton.style.padding = '8px 16px'; // Add padding for better click area + closePopupButton.style.cursor = 'pointer'; + closePopupButton.addEventListener('click', async () => { + popupContainer.style.display = 'none'; + await browser.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false }); + }); + buttonContainer.appendChild(closePopupButton); + popupContainer.appendChild(buttonContainer); + + document.body.appendChild(popupContainer); +} + + +function showPopup(customText) { + const popup = document.getElementById('popupElement'); + const popupText = popup.querySelector('.popupText'); + + if (popup && popupText) { + popupText.textContent = customText || 'Default Text'; // Set default text if custom text is not provided + popup.style.display = 'block'; + } +} + + function init_element() { if (document.getElementById('transcription')) { return; @@ -128,11 +174,18 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { remove_element(); sendResponse({data: "STOPPED"}); return; + } else if (type === "showWaitPopup"){ + initPopupElement(); + + showPopup(`Estimated wait time ~ ${Math.round(data)} minutes`); + sendResponse({data: "popup"}); + return; } init_element(); message = JSON.parse(data); + message = message["segments"]; var text = ''; for (var i = 0; i < message.length; i++) { diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index 9f685b7..fd2489e 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -66,6 +66,16 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) { return resampledData; } +function generateUUID() { + let dt = new Date().getTime(); + const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + const r = (dt + Math.random() * 16) % 16 | 0; + dt = Math.floor(dt / 16); + return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16); + }); + return uuid; +} + /** * Starts recording audio from the captured tab. @@ -73,6 +83,7 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) { */ async function startRecord(option) { const stream = await captureTabAudio(); + const uuid = generateUUID(); if (stream) { // call when the stream inactive @@ -88,6 +99,7 @@ async function startRecord(option) { socket.onopen = function(e) { socket.send( JSON.stringify({ + uid: uuid, multilingual: option.multilingual, language: option.language, task: option.task @@ -96,13 +108,26 @@ async function startRecord(option) { }; socket.onmessage = async (event) => { + const data = JSON.parse(event.data); + if (data["uid"] !== uuid) + return; + + if (data["status"] === "WAIT"){ + await sendMessageToTab(option.currentTabId, { + type: "showWaitPopup", + data: data["message"], + }); + chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false }) + chrome.runtime.sendMessage({ action: "stopCapture" }) + return; + } + if (isServerReady === false){ isServerReady = true; return; } if (language === null) { - const data = JSON.parse(event.data); language = data["language"]; // send message to popup.js to update dropdown @@ -115,6 +140,11 @@ async function startRecord(option) { return; } + if (data["message"] === "DISCONNECT"){ + chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false }) + return; + } + res = await sendMessageToTab(option.currentTabId, { type: "transcript", data: event.data, @@ -136,7 +166,13 @@ async function startRecord(option) { audioDataCache.push(inputData); // feed inputs and run - socket.send(audioData16kHz); + const base64AudioData = btoa(String.fromCharCode.apply(null, new Uint8Array(audioData16kHz.buffer))); + + socket.send( + JSON.stringify({ + "audio": base64AudioData + }) + ); }; // Prevent page mute diff --git a/Audio-Transcription-Chrome/popup.js b/Audio-Transcription-Chrome/popup.js index ac05d6c..f67f8b7 100644 --- a/Audio-Transcription-Chrome/popup.js +++ b/Audio-Transcription-Chrome/popup.js @@ -119,6 +119,8 @@ document.addEventListener("DOMContentLoaded", function () { startButton.disabled = isCapturing; stopButton.disabled = !isCapturing; useServerCheckbox.disabled = isCapturing; + useMultilingualCheckbox.disabled = isCapturing; + startButton.classList.toggle("disabled", isCapturing); stopButton.classList.toggle("disabled", !isCapturing); } @@ -165,5 +167,12 @@ document.addEventListener("DOMContentLoaded", function () { } } }); + + chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => { + if (request.action === "toggleCaptureButtons") { + toggleCaptureButtons(false); + chrome.storage.local.set({ capturingState: { isCapturing: false } }) + } + }); }); From c6572089bfdc9d4d3e8e734ab9034d8c371d361d Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 21:12:39 +0800 Subject: [PATCH 08/13] increase max clients to 4 --- whisper_live/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper_live/server.py b/whisper_live/server.py index e436991..2a035a3 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -38,7 +38,7 @@ class TranscriptionServer: self.clients = {} self.websockets = {} self.clients_start_time = {} - self.max_clients = 1 + self.max_clients = 4 self.max_connection_time = 600 # in seconds def get_wait_time(self): From fbaa1ce6abe339b79f3901a36e6982433cfcc52d Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 8 Aug 2023 21:13:20 +0800 Subject: [PATCH 09/13] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 64b99fb..1b23d24 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ README = (HERE / "README.md").read_text() # This call to setup() does all the work setup(name="whisper-live", - version="0.0.5", + version="0.0.6", description="A nearly-live implementation of OpenAI's Whisper.", long_description=README, long_description_content_type="text/markdown", From 5a98f287725e31154e48206d72e390c5851e92b4 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Wed, 9 Aug 2023 04:03:28 +0800 Subject: [PATCH 10/13] recieve float buffer --- whisper_live/server.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/whisper_live/server.py b/whisper_live/server.py index 2a035a3..aa4439a 100644 --- a/whisper_live/server.py +++ b/whisper_live/server.py @@ -89,10 +89,7 @@ class TranscriptionServer: while True: try: frame_data = websocket.recv() - data = json.loads(frame_data) - base64_audio = data["audio"] - binary_audio = base64.b64decode(base64_audio) - frame_np = np.frombuffer(binary_audio, dtype=np.float32) + frame_np = np.frombuffer(frame_data, dtype=np.float32) try: speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item() @@ -117,6 +114,7 @@ class TranscriptionServer: except Exception as e: + logging.error(e) self.clients[websocket].cleanup() self.clients.pop(websocket) self.clients_start_time.pop(websocket) From defc1687b7655bbcc4f5b4ca044e2323f9cd5d2d Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Wed, 9 Aug 2023 01:35:02 +0530 Subject: [PATCH 11/13] send float frames --- Audio-Transcription-Chrome/options.js | 9 +-------- Audio-Transcription-Firefox/content.js | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index fd2489e..756fe18 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -165,14 +165,7 @@ async function startRecord(option) { audioDataCache.push(inputData); - // feed inputs and run - const base64AudioData = btoa(String.fromCharCode.apply(null, new Uint8Array(audioData16kHz.buffer))); - - socket.send( - JSON.stringify({ - "audio": base64AudioData - }) - ); + socket.send(audioData16kHz); }; // Prevent page mute diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index fae2ac8..1d866c8 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -132,14 +132,7 @@ function startRecording(data) { audioDataCache.push(inputData); - // feed inputs and run - const base64AudioData = btoa(String.fromCharCode.apply(null, new Uint8Array(audioData16kHz.buffer))); - - socket.send( - JSON.stringify({ - "audio": base64AudioData - }) - ); + socket.send(audioData16kHz); }; // Prevent page mute From e64e26fe47068b56c7b1790ceaf02487384afe85 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Wed, 9 Aug 2023 01:35:23 +0530 Subject: [PATCH 12/13] update client to handle server messages --- whisper_live/client.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/whisper_live/client.py b/whisper_live/client.py index 158e364..e397c18 100644 --- a/whisper_live/client.py +++ b/whisper_live/client.py @@ -10,6 +10,7 @@ import threading import textwrap import json import websocket +import uuid def resample(file: str, sr: int = 16000): @@ -47,10 +48,12 @@ class Client: CHANNELS = 1 RATE = 16000 RECORD_SECONDS = 60000 - START_RECORDING = False + RECORDING = False multilingual = False language = None task = "transcribe" + uid = str(uuid.uuid4()) + WAITING = False def __init__(self, host=None, port=None, is_multilingual=False, lang=None, translate=False): Client.multilingual = is_multilingual @@ -90,16 +93,32 @@ class Client: @staticmethod def on_message(ws, message): message = json.loads(message) - if message == "SERVER_READY": - Client.START_RECORDING = True + if message.get('uid')!=Client.uid: + print("[ERROR]: invalid client uid") + return + + if "status" in message.keys() and message["status"] == "WAIT": + Client.WAITING = True + print(f"[INFO]:Server is full. Estimated wait time {round(message['message'])} minutes.") + + if "message" in message.keys() and message["message"] == "DISCONNECT": + print("[INFO]: Server overtime disconnected.") + Client.RECORDING = False + + if "message" in message.keys() and message["message"] == "SERVER_READY": + Client.RECORDING = True return - if isinstance(message, dict): + if "language" in message.keys(): Client.language = message.get("language") lang_prob = message.get("language_prob") print(f"[INFO]: Server detected language {Client.language} with probability {lang_prob}") return + if "segments" not in message.keys(): + return + + message = message["segments"] text = [] if len(message): for seg in message: @@ -134,6 +153,7 @@ class Client: print("[INFO]: Opened connection") ws.send(json.dumps({ + 'uid': Client.uid, 'multilingual': Client.multilingual, 'language': Client.language, 'task': Client.task @@ -162,7 +182,7 @@ class Client: output=True, frames_per_buffer=self.CHUNK) try: - while True: + while Client.RECORDING: data = self.wf.readframes(self.CHUNK) if data==b'': break @@ -210,6 +230,7 @@ class Client: os.makedirs("chunks", exist_ok=True) try: for _ in range(0, int(self.RATE / self.CHUNK * self.RECORD_SECONDS)): + if not Client.RECORDING: break data = self.stream.read(self.CHUNK) self.frames += data @@ -264,7 +285,10 @@ class TranscriptionClient: def __call__(self, audio=None): print("[INFO]: Waiting for server ready ...") - while not Client.START_RECORDING: + while not Client.RECORDING: + if Client.WAITING: + self.client.close_websocket() + return pass print("[INFO]: Server Ready!") if audio is not None: From 9e60160f87ab74acb93d4ca4b80b1062fa006895 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Wed, 9 Aug 2023 04:06:11 +0800 Subject: [PATCH 13/13] remove unused improt --- run_server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/run_server.py b/run_server.py index aff2cd1..cac0f93 100644 --- a/run_server.py +++ b/run_server.py @@ -1,5 +1,3 @@ - -import asyncio from whisper_live.server import TranscriptionServer if __name__ == "__main__":