add firefox extension

Audio-Transcription-Firefox/README.md

@@ -0,0 +1,33 @@
+# Audio Transcription Fox
+
+Audio Transcription is a Firefox extension that allows users to capture any audio playing on the current tab and transcribe it using OpenAI-whisper in real time. Users will have the option to do voice activity detection as well to not send audio to server when there is no speech.
+
+We use OpenAI-whisper model to process the audio continuously and send the transcription back to the client. We apply a few optimizations on top of OpenAI's implementation to improve performance and run it faster in a real-time manner. To this end, we used [faster-whisper](https://github.com/guillaumekln/faster-whisper) which is 4x faster than OpenAI's implementation.
+
+## Loading the Extension
+- Open the Mozilla Firefox browser.
+- Type ```about:debugging#/runtime/this-firefox``` in the address bar and press Enter.
+- Clone this repository
+- Click the Load temporary Add-on.
+- Browse to the location where you cloned the repository files and select the ```Audio Transcription Fox``` folder.
+- The extension should now be loaded and visible on the extensions page.
+
+
+## Real time transcription with OpenAI-whisper
+This Firefox extension allows you to send audio from your browser to a server for transcribing the audio in real time. 
+
+## Implementation Details
+
+### Capturing Audio
+To capture the audio in the current tab, we used the chrome `tabCapture` API to obtain a `MediaStream` object of the current tab.
+
+### Getting Started
+- Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live).
+- Just click on the Firefox Extension which should show 2 options
+  - **Start Capture** : Starts capturing the audio in the current tab and sends the captured audio to the server for transcription. This also creates an element to show the transcriptions recieved from the server on the current tab.
+  - **Stop Capture** - Stops capturing the audio.
+
+
+## Limitations
+This extension requires an internet connection to stream audio and receive transcriptions. The accuracy of the transcriptions may vary depending on the audio quality and the performance of the server-side transcription service. The extension may consume additional system resources while running, especially when streaming audio.
+

Audio-Transcription-Firefox/background.js

@@ -0,0 +1,18 @@
+browser.runtime.onMessage.addListener(function(request, sender, sendResponse) {
+  const { action, data } = request;
+  if (action === "transcript") {
+    browser.tabs.query({ active: true, currentWindow: true })
+      .then((tabs) => {
+        const tabId = tabs[0].id;
+        browser.tabs.sendMessage(tabId, { action: "show_transcript", data });
+      })
+      .catch((error) => {
+        console.error("Error retrieving active tab:", error);
+      });
+  }
+});
+
+
+
+
+  

Audio-Transcription-Firefox/content.js

@@ -0,0 +1,283 @@
+console.log("Content script injected.");
+let socket = null;
+let isCapturing = false;
+let mediaStream = null;
+let audioContext = null;
+let scriptProcessor = null; 
+
+/**
+ * Resamples the audio data to a target sample rate of 16kHz.
+ * @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data.
+ * @param {number} [origSampleRate=44100] - The original sample rate of the audio data.
+ * @returns {Float32Array} The resampled audio data at 16kHz.
+ */
+function resampleTo16kHZ(audioData, origSampleRate = 44100) {
+  // Convert the audio data to a Float32Array
+  const data = new Float32Array(audioData);
+
+  // Calculate the desired length of the resampled data
+  const targetLength = Math.round(data.length * (16000 / origSampleRate));
+
+  // Create a new Float32Array for the resampled data
+  const resampledData = new Float32Array(targetLength);
+
+  // Calculate the spring factor and initialize the first and last values
+  const springFactor = (data.length - 1) / (targetLength - 1);
+  resampledData[0] = data[0];
+  resampledData[targetLength - 1] = data[data.length - 1];
+
+  // Resample the audio data
+  for (let i = 1; i < targetLength - 1; i++) {
+    const index = i * springFactor;
+    const leftIndex = Math.floor(index).toFixed();
+    const rightIndex = Math.ceil(index).toFixed();
+    const fraction = index - leftIndex;
+    resampledData[i] = data[leftIndex] + (data[rightIndex] - data[leftIndex]) * fraction;
+  }
+
+  // Return the resampled data
+  return resampledData;
+}
+
+function startRecording() {
+    socket = new WebSocket("ws://localhost:9090/");
+    socket.onopen = function(e) { 
+      socket.send("handshake");
+    };
+
+    socket.onmessage = (event) => {
+      // console.log(event.data);
+      const data = event.data;
+      browser.runtime.sendMessage({ action: "transcript", data })
+          .catch(function(error) {
+            console.error("Error sending message:", error);
+          });
+    };
+
+  // Access the audio stream from the current tab
+  navigator.mediaDevices.getUserMedia({ audio: true })
+    .then(function(stream) {
+      // Create a new MediaRecorder instance
+      const audioDataCache = [];
+      audioContext = new AudioContext();
+      mediaStream = audioContext.createMediaStreamSource(stream);
+      recorder = audioContext.createScriptProcessor(4096, 1, 1);
+
+      recorder.onaudioprocess = async (event) => {
+        if (!audioContext || !isCapturing) return;
+
+        const inputData = event.inputBuffer.getChannelData(0);
+        const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate);
+
+        audioDataCache.push(inputData);
+
+        // feed inputs and run
+        socket.send(audioData16kHz);
+      };
+
+      // Prevent page mute
+      mediaStream.connect(recorder);
+      recorder.connect(audioContext.destination);
+    })
+}
+
+var elem_container = null;
+var elem_text = null;
+
+var segments = [];
+var text_segments = [];
+
+function init_element() {
+    if (document.getElementById('transcription')) {
+        return;
+    }
+
+    elem_container = document.createElement('div');
+    elem_container.id = "transcription";
+    elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;top:0px;position:absolute;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;';
+
+    for (var i = 0; i < 4; i++) {
+        elem_text = document.createElement('span');
+        elem_text.style.cssText = 'position: absolute;padding-left:16px;padding-right:16px;';
+        elem_text.id = "t" + i;
+        elem_container.appendChild(elem_text);
+
+        if (i == 3) {
+            elem_text.style.top = "-1000px"
+        }
+    }
+
+    document.body.appendChild(elem_container);
+
+    let x = 0;
+    let y = 0;
+
+    // Query the element
+    const ele = elem_container;
+
+    // Handle the mousedown event
+    // that's triggered when user drags the element
+    const mouseDownHandler = function (e) {
+        // Get the current mouse position
+        x = e.clientX;
+        y = e.clientY;
+
+        // Attach the listeners to `document`
+        document.addEventListener('mousemove', mouseMoveHandler);
+        document.addEventListener('mouseup', mouseUpHandler);
+    };
+
+    const mouseMoveHandler = function (e) {
+        // How far the mouse has been moved
+        const dx = e.clientX - x;
+        const dy = e.clientY - y;
+
+        // Set the position of element
+        ele.style.top = `${ele.offsetTop + dy}px`;
+        ele.style.left = `${ele.offsetLeft + dx}px`;
+
+        // Reassign the position of mouse
+        x = e.clientX;
+        y = e.clientY;
+    };
+
+    const mouseUpHandler = function () {
+        // Remove the handlers of `mousemove` and `mouseup`
+        document.removeEventListener('mousemove', mouseMoveHandler);
+        document.removeEventListener('mouseup', mouseUpHandler);
+    };
+
+    ele.addEventListener('mousedown', mouseDownHandler);
+}
+
+function getStyle(el,styleProp)
+{
+    var x = document.getElementById(el);
+    if (x.currentStyle)
+        var y = x.currentStyle[styleProp];
+    else if (window.getComputedStyle)
+        var y = document.defaultView.getComputedStyle(x,null).getPropertyValue(styleProp);
+    return y;
+}
+
+function get_lines(elem, line_height) {
+    var divHeight = elem.offsetHeight;
+    var lines = divHeight / line_height;
+
+    var original_text = elem.innerHTML;
+
+    var words = original_text.split(' ');
+    var segments = [];
+    var current_lines = 1;
+    var segment = '';
+    var segment_len = 0;
+    for (var i = 0; i < words.length; i++)
+    {
+        segment += words[i] + ' ';
+        elem.innerHTML = segment;
+        divHeight = elem.offsetHeight;
+
+        if ((divHeight / line_height) > current_lines) {
+            var line_segment = segment.substring(segment_len, segment.length - 1 - words[i].length - 1);
+            segments.push(line_segment);
+            segment_len += line_segment.length + 1;
+            current_lines++;
+        }
+    }
+
+    var line_segment = segment.substring(segment_len, segment.length - 1)
+    segments.push(line_segment);
+
+    elem.innerHTML = original_text;
+
+    return segments;
+
+}
+
+function remove_element() {
+    var elem = document.getElementById('transcription')
+    for (var i = 0; i < 4; i++) {
+        document.getElementById("t" + i).remove();
+    }
+    elem.remove()
+}
+
+
+browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
+  const { action, data } = request;
+  if (action === "startCapture") {
+      isCapturing = true;
+      startRecording();
+  } else if (action === "stopCapture") {
+    
+    isCapturing = false;
+    if (socket) {
+        socket.close();
+        socket = null;
+    }
+    
+    if (audioContext) {
+        audioContext.close();
+        audioContext = null;
+        mediaStream = null;
+        recorder = null;
+    }
+
+    
+    remove_element();
+
+  } else if (action === "show_transcript"){
+    if (!isCapturing) return;
+    init_element();    
+    message = JSON.parse(data);
+    
+    var text = '';
+    for (var i = 0; i < message.length; i++) {
+        text += message[i].text + ' ';
+    }
+    text = text.replace(/(\r\n|\n|\r)/gm, "");
+    
+    var elem = document.getElementById('t3');
+    elem.innerHTML = text;
+  
+    var line_height_style = getStyle('t3', 'line-height');
+    var line_height = parseInt(line_height_style.substring(0, line_height_style.length - 2));
+    var divHeight = elem.offsetHeight;
+    var lines = divHeight / line_height;
+  
+    text_segments = [];
+    text_segments = get_lines(elem, line_height);
+    
+    elem.innerHTML = '';
+  
+    if (text_segments.length > 2) {
+        for (var i = 0; i < 3; i++) {
+            document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
+        }
+    } else {
+        for (var i = 0; i < 3; i++) {
+            document.getElementById('t' + i).innerHTML = '';
+        }
+    }
+  
+    if (text_segments.length <= 2) {
+        for (var i = 0; i < text_segments.length; i++) {
+            document.getElementById('t' + i).innerHTML = text_segments[i];
+        }
+    } else {
+        for (var i = 0; i < 3; i++) {
+            document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
+        }
+    }
+  
+    for (var i = 1; i < 3; i++)
+    {
+        var parent_elem = document.getElementById('t' + (i - 1));
+        var elem = document.getElementById('t' + i);
+        elem.style.top = parent_elem.offsetHeight + parent_elem.offsetTop + 'px';
+    }
+  }
+  sendResponse({});
+});
+
+

Audio-Transcription-Firefox/manifest.json

@@ -0,0 +1,27 @@
+{
+    "manifest_version": 2,
+    "name": "Audio Recorder",
+    "version": "1.0",
+    "description": "Record audio from any webpage.",
+    "permissions": [
+      "activeTab",
+      "<all_urls>"
+    ],
+    "background": {
+      "scripts": ["background.js"],
+      "persistent": false
+    },
+    "browser_action": {
+      "default_popup": "popup.html",
+      "default_icon": "icon128.png"
+    },
+    "icons": {
+        "128":"icon128.png"
+        },
+    "content_scripts": [
+      {
+        "matches": ["<all_urls>"],
+        "js": ["content.js"]
+      }
+    ]
+  }

Audio-Transcription-Firefox/popup.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Audio Capture</title>
+  <script src="popup.js"></script>
+  <link rel="stylesheet" href="style.css" type="text/css">
+</head>
+<body>
+  <div class="header"><img src="./icon128.png"/> <h1>Audio Transcription</h1></div>
+  <div class="button-container">
+    <div class="button" id="startCapture">Start Capture</div>
+    <div class="button" id="stopCapture" disabled>Stop Capture</div>
+  </div>
+</body>
+</html>

Audio-Transcription-Firefox/popup.js

@@ -0,0 +1,38 @@
+document.addEventListener("DOMContentLoaded", function() {
+  var startButton = document.getElementById("startCapture");
+  var stopButton = document.getElementById("stopCapture");
+
+  startButton.addEventListener("click", function() {
+    browser.tabs.query({ active: true, currentWindow: true })
+      .then(function(tabs) {
+        browser.tabs.sendMessage(tabs[0].id, { action: "startCapture" });
+        toggleCaptureButtons(true);
+      })
+      .catch(function(error) {
+        console.error("Error sending startCapture message:", error);
+      });
+  });
+
+  stopButton.addEventListener("click", function() {
+    browser.tabs.query({ active: true, currentWindow: true })
+      .then(function(tabs) {
+        browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture" })
+          .then(function(response) {
+            console.log(response);
+            toggleCaptureButtons(false);
+          })
+          .catch(function(error) {
+            console.error("Error sending stopCapture message:", error);
+          });
+      })
+      .catch(function(error) {
+        console.error("Error querying active tab:", error);
+      });
+  });
+
+  // Function to toggle the capture buttons
+  function toggleCaptureButtons(isCapturing) {
+    startButton.disabled = isCapturing;
+    stopButton.disabled = !isCapturing;
+  }
+});

Audio-Transcription-Firefox/style.css

@@ -0,0 +1,103 @@
+.header {
+  display: flex;
+  align-items: center;
+  padding-bottom: 15px;
+  padding-left: 20px;
+  border-bottom: 2px solid darkred;
+}
+
+.header-title {
+  padding: 0 5px;
+}
+
+h1 {
+  font-size: 36px;
+}
+
+img {
+  height: 64px;
+  margin: 0 20px 0 0;
+}
+
+h2 {
+  font-size: 26px;
+}
+
+label {
+  font-size: 16px;
+}
+
+.inner {
+  margin-left: 40px;
+}
+
+.options-list {
+  padding: 0;
+  list-style: none;
+}
+
+.options-list li {
+  padding: 10px;
+}
+
+.time {
+  font-size: 16px;
+}
+
+.limit {
+  display: inline-block;
+  margin: 0;
+  font-size: 12px;
+}
+
+.radioChoice {
+  margin-left: 15px;
+}
+
+.button-container {
+  display: flex;
+  justify-content: space-between;
+  padding: 10px;
+}
+
+.button {
+  padding: 10px;
+  border: 2px solid darkred;
+  font-size: 16px;
+  font-weight: bold;
+  cursor: pointer;
+  white-space: nowrap;
+  width: 150px;
+  border-radius: 5px;
+}
+
+.disabled {
+  opacity: 0.6;
+  cursor: not-allowed;
+}
+
+.button:hover:not(:disabled) {
+  color: red;
+  background-color: darkred;
+}
+
+#save {
+  font-size: 16px;
+  margin-left: 50px;
+}
+
+#status {
+  color: red;
+  margin-top: 8px;
+  margin-left: 50px;
+  font-size: 14px;
+}
+
+#qualityLi {
+  display: none;
+}
+
+#maxTime {
+  width: 30px;
+  text-align: center;
+}