From 8e9baaea8fe207dcd3f9b8dcf66e8ab0357e042c Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 17 Jul 2023 22:06:59 +0530 Subject: [PATCH 1/4] add multilingual, task option to python client & server --- client.py | 21 +++++++++++++++++++++ server.py | 36 +++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/client.py b/client.py index 62dfd83..b38f7bd 100644 --- a/client.py +++ b/client.py @@ -18,6 +18,9 @@ CHANNELS = 1 RATE = 16000 RECORD_SECONDS = 60000 START_RECORDING = False +multilingual = False +language = None + def on_message(ws, message): @@ -53,7 +56,16 @@ def on_close(ws, close_status_code, close_msg): print("### websocket connection closed ###") def on_open(ws): + global multilingual, language, task + print(multilingual, language, task) + print("Opened connection") + ws.send(json.dumps({ + 'multilingual': multilingual[0], + 'language': language[0], + 'task': task + })) + class Client: @@ -226,13 +238,22 @@ if __name__=="__main__": parser.add_argument('--audio', type=str, help='audio file to transcribe') parser.add_argument('--host', default=None, type=str, help='websocket server address to connect to') parser.add_argument('--port', default=None, type=str, help='websocket server port to connect to') + parser.add_argument('--multilingual', action="store_true", help='use multilingual model') + parser.add_argument('--language', default=None, type=str, help='languages to use') + parser.add_argument( + '--task', default="transcribe", type=str, help='task transcribe/translate (translates from any to english)') opt = parser.parse_args() + print(opt) + multilingual=opt.multilingual, + language = opt.language, + task = opt.task c = Client(host=opt.host, port=opt.port) # while loop to wait for server to be ready print("Waiting for server ready ...") while not START_RECORDING: pass + print("Server Ready!") if os.name=='nt': os.system('cls') else: diff --git a/server.py b/server.py index f189214..67543bb 100644 --- a/server.py +++ b/server.py @@ -26,14 +26,20 @@ def recv_audio(websocket): Receive audio chunks from client in an infinite loop. """ global clients - client = ServeClient(websocket) + options = websocket.recv() + options = json.loads(options) + client = ServeClient( + websocket, + multilingual=options["multilingual"], + language=options["language"], + task=options["task"] + ) + clients[websocket] = client + while True: try: frame_data = websocket.recv() - if isinstance(frame_data, str): - logging.info(frame_data) - continue frame_np = np.frombuffer(frame_data, np.float32) clients[websocket].add_frames(frame_np) @@ -46,15 +52,16 @@ def recv_audio(websocket): class ServeClient: RATE = 16000 - def __init__(self, websocket, topic=None, device=None): - self.payload_size = struct.calcsize("Q") + def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None): self.data = b"" self.frames = b"" + self.language = language + self.task = task self.transcriber = WhisperModel( - "small.en", - device="cuda", + "small" if multilingual else "small.en", + device=device if device else "cuda", compute_type="float16", - local_files_only=False + local_files_only=False, ) # voice activity detection model @@ -83,9 +90,6 @@ class ServeClient: self.wrapper = textwrap.TextWrapper(width=50) self.pick_previous_segments = 2 - # setup mqtt - self.topic = topic - # threading self.websocket = websocket self.trans_thread = threading.Thread(target=self.speech_to_text) @@ -164,7 +168,13 @@ class ServeClient: initial_prompt = None # whisper transcribe with prompt - result = self.transcriber.transcribe(input_sample, initial_prompt=initial_prompt) + result = self.transcriber.transcribe( + input_sample, + initial_prompt=initial_prompt, + language=self.language, + task=self.task + ) + if len(result): self.t_start = None last_segment = self.update_segments(result, duration) From f6dce0e00897e443f5d2db8d4503f2f527babcd1 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 17 Jul 2023 22:13:03 +0530 Subject: [PATCH 2/4] add multilingual, language & task option in Chrome extension --- Audio-Transcription-Chrome/background.js | 9 +- Audio-Transcription-Chrome/options.js | 9 +- Audio-Transcription-Chrome/popup.html | 118 ++++++++++++++++++++++- Audio-Transcription-Chrome/popup.js | 66 +++++++++++-- Audio-Transcription-Chrome/style.css | 4 + 5 files changed, 196 insertions(+), 10 deletions(-) diff --git a/Audio-Transcription-Chrome/background.js b/Audio-Transcription-Chrome/background.js index 09de7dc..e21b04f 100644 --- a/Audio-Transcription-Chrome/background.js +++ b/Audio-Transcription-Chrome/background.js @@ -150,7 +150,14 @@ async function startCapture(options) { await sendMessageToTab(optionTab.id, { type: "start_capture", - data: { currentTabId: currentTab.id, host: options.host, port: options.port }, + data: { + currentTabId: currentTab.id, + host: options.host, + port: options.port, + multilingual: options.useMultilingual, + language: options.language, + task: options.task + }, }); } else { console.log("No Audio"); diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index ad7dfb1..ac2862c 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -79,11 +79,16 @@ async function startRecord(option) { stream.oninactive = () => { window.close(); }; - const socket = new WebSocket(`ws://${option.host}:${option.port}/`); let isServerReady = false; socket.onopen = function(e) { - socket.send("handshake"); + socket.send( + JSON.stringify({ + multilingual: option.multilingual, + language: option.language, + task: option.task + }) + ); }; socket.onmessage = async (event) => { diff --git a/Audio-Transcription-Chrome/popup.html b/Audio-Transcription-Chrome/popup.html index 5777929..4f2b6ec 100644 --- a/Audio-Transcription-Chrome/popup.html +++ b/Audio-Transcription-Chrome/popup.html @@ -15,5 +15,121 @@ +
+ + +
+
+ + +
+
+ + +
- \ No newline at end of file + diff --git a/Audio-Transcription-Chrome/popup.js b/Audio-Transcription-Chrome/popup.js index 20b061c..87507a0 100644 --- a/Audio-Transcription-Chrome/popup.js +++ b/Audio-Transcription-Chrome/popup.js @@ -4,6 +4,11 @@ document.addEventListener("DOMContentLoaded", function () { const stopButton = document.getElementById("stopCapture"); const useServerCheckbox = document.getElementById("useServerCheckbox"); + const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox'); + const languageDropdown = document.getElementById('languageDropdown'); + const taskDropdown = document.getElementById('taskDropdown'); + let selectedLanguage = undefined; + let selectedTask = taskDropdown.value; // Add click event listeners to the buttons startButton.addEventListener("click", startCapture); @@ -25,6 +30,28 @@ document.addEventListener("DOMContentLoaded", function () { } }); + chrome.storage.local.get("useMultilingualModelState", ({ useMultilingualModelState }) => { + if (useMultilingualModelState !== undefined) { + useMultilingualCheckbox.checked = useMultilingualModelState; + languageDropdown.disabled = !useMultilingualModelState; + taskDropdown.disabled = !useMultilingualModelState; + } + }); + + chrome.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => { + if (storedLanguage !== undefined) { + languageDropdown.value = storedLanguage; + selectedLanguage = storedLanguage; + } + }); + + chrome.storage.local.get("selectedTask", ({ selectedTask: storedTask }) => { + if (storedTask !== undefined) { + taskDropdown.value = storedTask; + selectedTask = storedTask; + } + }); + // Function to handle the start capture button click event async function startCapture() { // Ignore click if the button is disabled @@ -49,12 +76,17 @@ document.addEventListener("DOMContentLoaded", function () { action: "startCapture", tabId: currentTab.id, host: host, - port: port }, () => { - // Update capturing state in storage and toggle the buttons - chrome.storage.local.set({ capturingState: { isCapturing: true } }, () => { - toggleCaptureButtons(true); - }); - }); + port: port, + useMultilingual: useMultilingualCheckbox.checked, + language: selectedLanguage, + task: selectedTask + }, () => { + // Update capturing state in storage and toggle the buttons + chrome.storage.local.set({ capturingState: { isCapturing: true } }, () => { + toggleCaptureButtons(true); + }); + } + ); } // Function to handle the stop capture button click event @@ -96,4 +128,26 @@ document.addEventListener("DOMContentLoaded", function () { const useServerState = useServerCheckbox.checked; chrome.storage.local.set({ useServerState }); }); + + useMultilingualCheckbox.addEventListener('change', function() { + const useMultilingualModelState = useMultilingualCheckbox.checked; + if (useMultilingualModelState) { + languageDropdown.disabled = false; + taskDropdown.disabled = false; + } else { + languageDropdown.disabled = true; + taskDropdown.disabled = true; + } + chrome.storage.local.set({ useMultilingualModelState }); + }); + + languageDropdown.addEventListener('change', function() { + selectedLanguage = languageDropdown.value; + chrome.storage.local.set({ selectedLanguage }); + }); + + taskDropdown.addEventListener('change', function() { + selectedTask = taskDropdown.value; + chrome.storage.local.set({ selectedTask }); + }); }); diff --git a/Audio-Transcription-Chrome/style.css b/Audio-Transcription-Chrome/style.css index f8bba91..dc6f27b 100644 --- a/Audio-Transcription-Chrome/style.css +++ b/Audio-Transcription-Chrome/style.css @@ -105,3 +105,7 @@ label { .checkbox-container { padding: 10px; } + +.dropdown-container { + padding: 10px; +} From 82a4c93019a746d5b9b2002e09cff509ed71f788 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 17 Jul 2023 22:13:34 +0530 Subject: [PATCH 3/4] add multilingual, language & task option in Firefox extension --- Audio-Transcription-Firefox/content.js | 14 ++- Audio-Transcription-Firefox/popup.html | 116 +++++++++++++++++++++++++ Audio-Transcription-Firefox/popup.js | 68 ++++++++++++++- Audio-Transcription-Firefox/style.css | 4 + 4 files changed, 194 insertions(+), 8 deletions(-) diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index eb8a32a..61a99fa 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -38,10 +38,16 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) { return resampledData; } -function startRecording(host, port) { - socket = new WebSocket(`ws://${host}:${port}/`); +function startRecording(data) { + socket = new WebSocket(`ws://${data.host}:${data.port}/`); socket.onopen = function(e) { - socket.send("handshake"); + socket.send( + JSON.stringify({ + multilingual: data.useMultilingual, + language: data.language, + task: data.task + }) + ); }; let isServerReady = false; @@ -209,7 +215,7 @@ browser.runtime.onMessage.addListener((request, sender, sendResponse) => { const { action, data } = request; if (action === "startCapture") { isCapturing = true; - startRecording(data.host, data.port); + startRecording(data); } else if (action === "stopCapture") { isCapturing = false; diff --git a/Audio-Transcription-Firefox/popup.html b/Audio-Transcription-Firefox/popup.html index 3766564..7393898 100644 --- a/Audio-Transcription-Firefox/popup.html +++ b/Audio-Transcription-Firefox/popup.html @@ -15,5 +15,121 @@ +
+ + +
+
+ + +
+
+ + +
\ No newline at end of file diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js index 1ef684d..905f3d3 100644 --- a/Audio-Transcription-Firefox/popup.js +++ b/Audio-Transcription-Firefox/popup.js @@ -1,11 +1,17 @@ document.addEventListener("DOMContentLoaded", function() { - var startButton = document.getElementById("startCapture"); - var stopButton = document.getElementById("stopCapture"); + const startButton = document.getElementById("startCapture"); + const stopButton = document.getElementById("stopCapture"); const useServerCheckbox = document.getElementById("useServerCheckbox"); + const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox'); + const languageDropdown = document.getElementById('languageDropdown'); + const taskDropdown = document.getElementById('taskDropdown'); + let selectedLanguage = undefined; + let selectedTask = taskDropdown.value; + browser.storage.local.get("capturingState") .then(function(result) { - var capturingState = result.capturingState; + const capturingState = result.capturingState; if (capturingState && capturingState.isCapturing) { toggleCaptureButtons(true); } else { @@ -26,6 +32,28 @@ document.addEventListener("DOMContentLoaded", function() { } }); + browser.storage.local.get("useMultilingualModelState", ({ useMultilingualModelState }) => { + if (useMultilingualModelState !== undefined) { + useMultilingualCheckbox.checked = useMultilingualModelState; + languageDropdown.disabled = !useMultilingualModelState; + taskDropdown.disabled = !useMultilingualModelState; + } + }); + + browser.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => { + if (storedLanguage !== undefined) { + languageDropdown.value = storedLanguage; + selectedLanguage = storedLanguage; + } + }); + + browser.storage.local.get("selectedTask", ({ selectedTask: storedTask }) => { + if (storedTask !== undefined) { + taskDropdown.value = storedTask; + selectedTask = storedTask; + } + }); + startButton.addEventListener("click", function() { let host = "localhost"; let port = "9090"; @@ -39,7 +67,17 @@ document.addEventListener("DOMContentLoaded", function() { browser.tabs.query({ active: true, currentWindow: true }) .then(function(tabs) { browser.tabs.sendMessage( - tabs[0].id, { action: "startCapture", data: {host, port} }); + tabs[0].id, + { + action: "startCapture", + data: { + host: host, + port: port, + useMultilingual: useMultilingualCheckbox.checked, + language: selectedLanguage, + task: selectedTask + } + }); toggleCaptureButtons(true); browser.storage.local.set({ capturingState: { isCapturing: true } }) .catch(function(error) { @@ -85,4 +123,26 @@ document.addEventListener("DOMContentLoaded", function() { const useServerState = useServerCheckbox.checked; browser.storage.local.set({ useServerState }); }); + + useMultilingualCheckbox.addEventListener('change', function() { + const useMultilingualModelState = useMultilingualCheckbox.checked; + if (useMultilingualModelState) { + languageDropdown.disabled = false; + taskDropdown.disabled = false; + } else { + languageDropdown.disabled = true; + taskDropdown.disabled = true; + } + browser.storage.local.set({ useMultilingualModelState }); + }); + + languageDropdown.addEventListener('change', function() { + selectedLanguage = languageDropdown.value; + browser.storage.local.set({ selectedLanguage }); + }); + + taskDropdown.addEventListener('change', function() { + selectedTask = taskDropdown.value; + browser.storage.local.set({ selectedTask }); + }); }); diff --git a/Audio-Transcription-Firefox/style.css b/Audio-Transcription-Firefox/style.css index f8bba91..6e743dc 100644 --- a/Audio-Transcription-Firefox/style.css +++ b/Audio-Transcription-Firefox/style.css @@ -105,3 +105,7 @@ label { .checkbox-container { padding: 10px; } + +.dropdown-container { + padding: 10px; +} \ No newline at end of file From a84da4f79c3914acad0ee0978678e353eaeb0ca2 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 17 Jul 2023 23:30:47 +0530 Subject: [PATCH 4/4] update docs --- Audio-Transcription-Chrome/README.md | 10 ++++++++++ Audio-Transcription-Firefox/README.md | 9 +++++++++ README.md | 12 ++++++++---- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/Audio-Transcription-Chrome/README.md b/Audio-Transcription-Chrome/README.md index 4e5fa8c..0d40290 100644 --- a/Audio-Transcription-Chrome/README.md +++ b/Audio-Transcription-Chrome/README.md @@ -23,6 +23,13 @@ This Chrome extension allows you to send audio from your browser to a server for ### Capturing Audio To capture the audio in the current tab, we used the chrome `tabCapture` API to obtain a `MediaStream` object of the current tab. +### Options +When using the Audio Transcription extension, you have the following options: + - **Use Collabora Server**: We provide a demo server which runs the whisper small model. + - **Use Multilingual Model**: Enable this option to utilize the multilingual capabilities of OpenAI-whisper. + - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper. + - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English. + ### Getting Started - Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live). - Just click on the Chrome Extension which should show 2 options @@ -33,3 +40,6 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to ## Limitations This extension requires an internet connection to stream audio and receive transcriptions. The accuracy of the transcriptions may vary depending on the audio quality and the performance of the server-side transcription service. The extension may consume additional system resources while running, especially when streaming audio. +## Note +The extension relies on a properly running transcription server with multilingual support. Please follow the server documentation for setup and configuration. + diff --git a/Audio-Transcription-Firefox/README.md b/Audio-Transcription-Firefox/README.md index a24c14d..52e4ceb 100644 --- a/Audio-Transcription-Firefox/README.md +++ b/Audio-Transcription-Firefox/README.md @@ -21,6 +21,13 @@ This Firefox extension allows you to send audio from your browser to a server fo ### Capturing Audio To capture the audio in the current tab, we used the chrome `tabCapture` API to obtain a `MediaStream` object of the current tab. +### Options +When using the Audio Transcription extension, you have the following options: + - **Use Collabora Server**: We provide a demo server which runs the whisper small model. + - **Use Multilingual Model**: Enable this option to utilize the multilingual capabilities of OpenAI-whisper. + - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper. + - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English. + ### Getting Started - Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live). - Just click on the Firefox Extension which should show 2 options @@ -31,3 +38,5 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to ## Limitations This extension requires an internet connection to stream audio and receive transcriptions. The accuracy of the transcriptions may vary depending on the audio quality and the performance of the server-side transcription service. The extension may consume additional system resources while running, especially when streaming audio. +## Note +The extension relies on a properly running transcription server with multilingual support. Please follow the server documentation for setup and configuration. diff --git a/README.md b/README.md index 8bf582d..dba34fd 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ A nearly-live implementation of OpenAI's Whisper. This project is a real-time transcription application that uses the OpenAI Whisper model to convert speech input into text output. It can be used to transcribe both live audio input from microphone and pre-recorded audio files. -Unlike traditional speech recognition systems that rely on continuous audio streaming, we use [voice activity detection (VAD)](https://github.com/snakers4/silero-vad) to detect the presence of speech and only send the audio data to whisper when speech is detected. This helps to reduce the amount of data sent to the API and improves the accuracy of the transcription output. +Unlike traditional speech recognition systems that rely on continuous audio streaming, we use [voice activity detection (VAD)](https://github.com/snakers4/silero-vad) to detect the presence of speech and only send the audio data to whisper when speech is detected. This helps to reduce the amount of data sent to the whisper model and improves the accuracy of the transcription output. ## Installation - Install PyAudio and ffmpeg @@ -30,13 +30,17 @@ Unlike traditional speech recognition systems that rely on continuous audio stre - On the client side - To transcribe an audio file: ```bash - python client.py --audio "audio.wav" --host "localhost" --port "9090" + python client.py --audio "audio.wav" --host "localhost" --port "9090" --multilingual --language "hi" --task "transcribe" + "translate" ``` + This command transcribes the specified audio file (audio.wav) using the Whisper model. It connects to the server running on localhost at port 9090. It also enables the multilingual feature, allowing transcription in multiple languages. The --language flag specifies the target language for transcription, in this case, Hindi ("hi"). The --task flag is set to "transcribe" to indicate that transcription is the desired task. Also, --task can be set to "translate" to translate source language to English. - To transcribe from microphone: ```bash - python client.py --host "localhost" --port "9090" + python client.py --host "localhost" --port "9090" --multilingual --language "en" --task "transcribe" ``` + This command captures audio from the microphone and sends it to the server for transcription. It uses the same options as the previous command, enabling the multilingual feature and specifying the target language and task. + ## Transcribe audio from browser - Run the server @@ -63,7 +67,7 @@ This would start the websocket server on port ```9090```. ``` ## Future Work -- [ ] Update Documentation. +- [x] Update Documentation. - [x] Keep only a single server implementation i.e. websockets and get rid of the socket implementation in ```server.py```. Also, update ```client.py``` to websockets-client implemenation. - [ ] Add translation to other languages on top of transcription.