diff --git a/Audio-Transcription-Chrome/background.js b/Audio-Transcription-Chrome/background.js index e21b04f..99848d7 100644 --- a/Audio-Transcription-Chrome/background.js +++ b/Audio-Transcription-Chrome/background.js @@ -195,6 +195,12 @@ chrome.runtime.onMessage.addListener((message) => { startCapture(message); } else if (message.action === "stopCapture") { stopCapture(); + } else if (message.action === "updateSelectedLanguage") { + console.log("Selected language"); + console.log(message.detectedLanguage); + const detectedLanguage = message.detectedLanguage; + chrome.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage }); + chrome.storage.local.set({ selectedLanguage: detectedLanguage }); } }); diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index ac2862c..316d5b6 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -81,6 +81,7 @@ async function startRecord(option) { }; const socket = new WebSocket(`ws://${option.host}:${option.port}/`); let isServerReady = false; + let language = option.language; socket.onopen = function(e) { socket.send( JSON.stringify({ @@ -92,12 +93,25 @@ async function startRecord(option) { }; socket.onmessage = async (event) => { - console.log(event.data); if (isServerReady === false){ isServerReady = true; return; } + if (language === null) { + const data = JSON.parse(event.data); + language = data["language"]; + + // send message to popup.js to update dropdown + // console.log(language); + chrome.runtime.sendMessage({ + action: "updateSelectedLanguage", + detectedLanguage: language, + }); + + return; + } + res = await sendMessageToTab(option.currentTabId, { type: "transcript", data: event.data, diff --git a/Audio-Transcription-Chrome/popup.js b/Audio-Transcription-Chrome/popup.js index 87507a0..ac05d6c 100644 --- a/Audio-Transcription-Chrome/popup.js +++ b/Audio-Transcription-Chrome/popup.js @@ -7,7 +7,7 @@ document.addEventListener("DOMContentLoaded", function () { const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox'); const languageDropdown = document.getElementById('languageDropdown'); const taskDropdown = document.getElementById('taskDropdown'); - let selectedLanguage = undefined; + let selectedLanguage = null; let selectedTask = taskDropdown.value; // Add click event listeners to the buttons @@ -142,7 +142,11 @@ document.addEventListener("DOMContentLoaded", function () { }); languageDropdown.addEventListener('change', function() { - selectedLanguage = languageDropdown.value; + if (languageDropdown.value === "") { + selectedLanguage = null; + } else { + selectedLanguage = languageDropdown.value; + } chrome.storage.local.set({ selectedLanguage }); }); @@ -150,4 +154,16 @@ document.addEventListener("DOMContentLoaded", function () { selectedTask = taskDropdown.value; chrome.storage.local.set({ selectedTask }); }); + + chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => { + if (request.action === "updateSelectedLanguage") { + const detectedLanguage = request.detectedLanguage; + + if (detectedLanguage) { + languageDropdown.value = detectedLanguage; + chrome.storage.local.set({ selectedLanguage: detectedLanguage }); + } + } + }); + }); diff --git a/Audio-Transcription-Firefox/background.js b/Audio-Transcription-Firefox/background.js index 9d3ea0c..4c57a84 100644 --- a/Audio-Transcription-Firefox/background.js +++ b/Audio-Transcription-Firefox/background.js @@ -10,5 +10,12 @@ browser.runtime.onMessage.addListener(function(request, sender, sendResponse) { console.error("Error retrieving active tab:", error); }); } + if (action === "updateSelectedLanguage") { + const detectedLanguage = data; + if (detectedLanguage) { + browser.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage }); + browser.storage.local.set({ selectedLanguage: detectedLanguage }); + } + } }); diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index 61a99fa..5a6391e 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -2,7 +2,8 @@ let socket = null; let isCapturing = false; let mediaStream = null; let audioContext = null; -let scriptProcessor = null; +let scriptProcessor = null; +let language = null; /** * Resamples the audio data to a target sample rate of 16kHz. @@ -40,6 +41,7 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) { function startRecording(data) { socket = new WebSocket(`ws://${data.host}:${data.port}/`); + language = data.language; socket.onopen = function(e) { socket.send( JSON.stringify({ @@ -56,7 +58,19 @@ function startRecording(data) { isServerReady = true; return; } - const data = event.data; + + if (language === null ){ + const data = JSON.parse(event.data); + language = data["language"]; + + browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language }) + .catch(function(error) { + console.error("Error sending message:", error); + }); + return + } + + const data = event.data;; browser.runtime.sendMessage({ action: "transcript", data }) .catch(function(error) { console.error("Error sending message:", error); diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js index 905f3d3..70923c3 100644 --- a/Audio-Transcription-Firefox/popup.js +++ b/Audio-Transcription-Firefox/popup.js @@ -6,7 +6,7 @@ document.addEventListener("DOMContentLoaded", function() { const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox'); const languageDropdown = document.getElementById('languageDropdown'); const taskDropdown = document.getElementById('taskDropdown'); - let selectedLanguage = undefined; + let selectedLanguage = null; let selectedTask = taskDropdown.value; browser.storage.local.get("capturingState") @@ -137,7 +137,11 @@ document.addEventListener("DOMContentLoaded", function() { }); languageDropdown.addEventListener('change', function() { - selectedLanguage = languageDropdown.value; + if (languageDropdown.value === "") { + selectedLanguage = null; + } else { + selectedLanguage = languageDropdown.value; + } browser.storage.local.set({ selectedLanguage }); }); @@ -145,4 +149,16 @@ document.addEventListener("DOMContentLoaded", function() { selectedTask = taskDropdown.value; browser.storage.local.set({ selectedTask }); }); + + browser.runtime.onMessage.addListener((request, sender, sendResponse) => { + if (request.action === "updateSelectedLanguage") { + const detectedLanguage = request.detectedLanguage; + + if (detectedLanguage) { + languageDropdown.value = detectedLanguage; + selectedLanguage = detectedLanguage; + browser.storage.local.set({ selectedLanguage }); + } + } + }); }); diff --git a/client.py b/client.py index b38f7bd..f912beb 100644 --- a/client.py +++ b/client.py @@ -24,11 +24,18 @@ language = None def on_message(ws, message): - global START_RECORDING + global START_RECORDING, language message = json.loads(message) if message == "SERVER_READY": START_RECORDING = True return + + if isinstance(message, dict): + language = message.get("language") + lang_prob = message.get("language_prob") + print(f"Server detected language {language} with probability {lang_prob}") + return + text = [] if len(message): for seg in message: @@ -66,7 +73,6 @@ def on_open(ws): 'task': task })) - class Client: def __init__(self, host=None, port=None): @@ -254,10 +260,6 @@ if __name__=="__main__": while not START_RECORDING: pass print("Server Ready!") - if os.name=='nt': - os.system('cls') - else: - os.system('clear') if opt.audio is not None: resampled_file = resample(opt.audio) diff --git a/server.py b/server.py index 483e5c4..9879b9f 100644 --- a/server.py +++ b/server.py @@ -140,6 +140,24 @@ class ServeClient: """ Process audio stream in an infinite loop. """ + # detect language + if self.language is None: + # wait for 30s of audio + while self.frames_np is None or self.frames_np.shape[0] < 30*self.RATE: + time.sleep(1) + input_bytes = self.frames_np[-30*self.RATE:].copy() + self.frames_np = None + duration = input_bytes.shape[0] / self.RATE + + self.language, lang_prob = self.transcriber.transcribe( + input_bytes, + initial_prompt=None, + language=self.language, + task=self.task + ) + logging.info(f"Detected language {self.language} with probability {lang_prob}") + self.websocket.send(json.dumps({"language": self.language, "language_prob": lang_prob})) + while True: if self.exit: logging.info("Exiting speech to text thread")