Merge pull request #26 from makaveli10/auto_detect_lang

Auto detect language.

Audio-Transcription-Chrome/background.js

@@ -195,6 +195,12 @@ chrome.runtime.onMessage.addListener((message) => {
     startCapture(message);
   } else if (message.action === "stopCapture") {
     stopCapture();
+  } else if (message.action === "updateSelectedLanguage") {
+    console.log("Selected language");
+    console.log(message.detectedLanguage);
+    const detectedLanguage = message.detectedLanguage;
+    chrome.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage });
+    chrome.storage.local.set({ selectedLanguage: detectedLanguage });
   }
 });
 

Audio-Transcription-Chrome/options.js

@@ -81,6 +81,7 @@ async function startRecord(option) {
     };
     const socket = new WebSocket(`ws://${option.host}:${option.port}/`);
     let isServerReady = false;
+    let language = option.language;
     socket.onopen = function(e) { 
       socket.send(
         JSON.stringify({
@@ -92,12 +93,25 @@ async function startRecord(option) {
     };
 
     socket.onmessage = async (event) => {
-      console.log(event.data);
       if (isServerReady === false){
         isServerReady = true;
         return;
       }
       
+      if (language === null) {
+        const data = JSON.parse(event.data);
+        language = data["language"];
+        
+        // send message to popup.js to update dropdown
+        // console.log(language);
+        chrome.runtime.sendMessage({
+          action: "updateSelectedLanguage",
+          detectedLanguage: language,
+        });
+
+        return;
+      }
+
       res = await sendMessageToTab(option.currentTabId, {
         type: "transcript",
         data: event.data,

Audio-Transcription-Chrome/popup.js

@@ -7,7 +7,7 @@ document.addEventListener("DOMContentLoaded", function () {
   const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox');
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
-  let selectedLanguage = undefined;
+  let selectedLanguage = null;
   let selectedTask = taskDropdown.value;
 
   // Add click event listeners to the buttons
@@ -142,7 +142,11 @@ document.addEventListener("DOMContentLoaded", function () {
   });
 
   languageDropdown.addEventListener('change', function() {
-    selectedLanguage = languageDropdown.value;
+    if (languageDropdown.value === "") {
+      selectedLanguage = null;
+    } else {
+      selectedLanguage = languageDropdown.value;
+    }
     chrome.storage.local.set({ selectedLanguage });
   });
 
@@ -150,4 +154,16 @@ document.addEventListener("DOMContentLoaded", function () {
     selectedTask = taskDropdown.value;
     chrome.storage.local.set({ selectedTask });
   });
+
+  chrome.runtime.onMessage.addListener(async (request, sender, sendResponse) => {
+    if (request.action === "updateSelectedLanguage") {
+      const detectedLanguage = request.detectedLanguage;
+  
+      if (detectedLanguage) {
+        languageDropdown.value = detectedLanguage;
+        chrome.storage.local.set({ selectedLanguage: detectedLanguage });
+      }
+    }
+  });
+  
 });

Audio-Transcription-Firefox/background.js

@@ -10,5 +10,12 @@ browser.runtime.onMessage.addListener(function(request, sender, sendResponse) {
         console.error("Error retrieving active tab:", error);
       });
   }
+  if (action === "updateSelectedLanguage") {
+    const detectedLanguage = data;
+    if (detectedLanguage) {
+      browser.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage });
+      browser.storage.local.set({ selectedLanguage: detectedLanguage });
+    }
+  }
 });
 

Audio-Transcription-Firefox/content.js

@@ -2,7 +2,8 @@ let socket = null;
 let isCapturing = false;
 let mediaStream = null;
 let audioContext = null;
-let scriptProcessor = null; 
+let scriptProcessor = null;
+let language = null;
 
 /**
  * Resamples the audio data to a target sample rate of 16kHz.
@@ -40,6 +41,7 @@ function resampleTo16kHZ(audioData, origSampleRate = 44100) {
 
 function startRecording(data) {
     socket = new WebSocket(`ws://${data.host}:${data.port}/`);
+    language = data.language;
     socket.onopen = function(e) { 
       socket.send(
         JSON.stringify({
@@ -56,7 +58,19 @@ function startRecording(data) {
         isServerReady = true;
         return;
       }
-      const data = event.data;
+
+      if (language === null ){
+        const data = JSON.parse(event.data);
+        language = data["language"];
+
+        browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language })
+          .catch(function(error) {
+            console.error("Error sending message:", error);
+          });
+        return
+      }
+
+      const data = event.data;;
       browser.runtime.sendMessage({ action: "transcript", data })
           .catch(function(error) {
             console.error("Error sending message:", error);

Audio-Transcription-Firefox/popup.js

@@ -6,7 +6,7 @@ document.addEventListener("DOMContentLoaded", function() {
   const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox');
   const languageDropdown = document.getElementById('languageDropdown');
   const taskDropdown = document.getElementById('taskDropdown');
-  let selectedLanguage = undefined;
+  let selectedLanguage = null;
   let selectedTask = taskDropdown.value;
 
   browser.storage.local.get("capturingState")
@@ -137,7 +137,11 @@ document.addEventListener("DOMContentLoaded", function() {
   });
 
   languageDropdown.addEventListener('change', function() {
-    selectedLanguage = languageDropdown.value;
+    if (languageDropdown.value === "") {
+      selectedLanguage = null;
+    } else {
+      selectedLanguage = languageDropdown.value;
+    }
     browser.storage.local.set({ selectedLanguage });
   });
 
@@ -145,4 +149,16 @@ document.addEventListener("DOMContentLoaded", function() {
     selectedTask = taskDropdown.value;
     browser.storage.local.set({ selectedTask });
   });
+
+  browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
+    if (request.action === "updateSelectedLanguage") {
+      const detectedLanguage = request.detectedLanguage;
+  
+      if (detectedLanguage) {
+        languageDropdown.value = detectedLanguage;
+        selectedLanguage = detectedLanguage;
+        browser.storage.local.set({ selectedLanguage });
+      }
+    }
+  });
 });

client.py

@@ -24,11 +24,18 @@ language = None
 
 
 def on_message(ws, message):
-    global START_RECORDING
+    global START_RECORDING, language
     message = json.loads(message)
     if message == "SERVER_READY":
         START_RECORDING = True
         return
+
+    if isinstance(message, dict):
+        language = message.get("language")
+        lang_prob = message.get("language_prob")
+        print(f"Server detected language {language} with probability {lang_prob}")
+        return
+
     text = []
     if len(message):
         for seg in message:
@@ -66,7 +73,6 @@ def on_open(ws):
         'task': task
     }))
 
-    
 
 class Client:
     def __init__(self, host=None, port=None):
@@ -254,10 +260,6 @@ if __name__=="__main__":
     while not START_RECORDING:
         pass
     print("Server Ready!")
-    if os.name=='nt':
-        os.system('cls')
-    else:
-        os.system('clear')
 
     if opt.audio is not None:
         resampled_file = resample(opt.audio)

server.py

@@ -140,6 +140,24 @@ class ServeClient:
         """
         Process audio stream in an infinite loop.
         """
+        # detect language
+        if self.language is None:
+            # wait for 30s of audio
+            while self.frames_np is None or self.frames_np.shape[0] < 30*self.RATE:
+                time.sleep(1)
+            input_bytes = self.frames_np[-30*self.RATE:].copy()
+            self.frames_np = None
+            duration = input_bytes.shape[0] / self.RATE
+
+            self.language, lang_prob = self.transcriber.transcribe(
+                    input_bytes, 
+                    initial_prompt=None,
+                    language=self.language,
+                    task=self.task
+                )
+            logging.info(f"Detected language {self.language} with probability {lang_prob}")
+            self.websocket.send(json.dumps({"language": self.language, "language_prob": lang_prob}))
+
         while True:
             if self.exit:
                 logging.info("Exiting speech to text thread")