From 2825bf4fe58afd81b15cba5be1d16a31f4a14abe Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Fri, 16 Jun 2023 19:46:56 +0530 Subject: [PATCH] add firefox extension --- Audio-Transcription-Firefox/README.md | 33 +++ Audio-Transcription-Firefox/background.js | 18 ++ Audio-Transcription-Firefox/content.js | 283 ++++++++++++++++++++++ Audio-Transcription-Firefox/icon128.png | Bin 0 -> 3054 bytes Audio-Transcription-Firefox/manifest.json | 27 +++ Audio-Transcription-Firefox/popup.html | 15 ++ Audio-Transcription-Firefox/popup.js | 38 +++ Audio-Transcription-Firefox/style.css | 103 ++++++++ 8 files changed, 517 insertions(+) create mode 100644 Audio-Transcription-Firefox/README.md create mode 100644 Audio-Transcription-Firefox/background.js create mode 100644 Audio-Transcription-Firefox/content.js create mode 100644 Audio-Transcription-Firefox/icon128.png create mode 100644 Audio-Transcription-Firefox/manifest.json create mode 100644 Audio-Transcription-Firefox/popup.html create mode 100644 Audio-Transcription-Firefox/popup.js create mode 100644 Audio-Transcription-Firefox/style.css diff --git a/Audio-Transcription-Firefox/README.md b/Audio-Transcription-Firefox/README.md new file mode 100644 index 0000000..af8a521 --- /dev/null +++ b/Audio-Transcription-Firefox/README.md @@ -0,0 +1,33 @@ +# Audio Transcription Fox + +Audio Transcription is a Firefox extension that allows users to capture any audio playing on the current tab and transcribe it using OpenAI-whisper in real time. Users will have the option to do voice activity detection as well to not send audio to server when there is no speech. + +We use OpenAI-whisper model to process the audio continuously and send the transcription back to the client. We apply a few optimizations on top of OpenAI's implementation to improve performance and run it faster in a real-time manner. To this end, we used [faster-whisper](https://github.com/guillaumekln/faster-whisper) which is 4x faster than OpenAI's implementation. + +## Loading the Extension +- Open the Mozilla Firefox browser. +- Type ```about:debugging#/runtime/this-firefox``` in the address bar and press Enter. +- Clone this repository +- Click the Load temporary Add-on. +- Browse to the location where you cloned the repository files and select the ```Audio Transcription Fox``` folder. +- The extension should now be loaded and visible on the extensions page. + + +## Real time transcription with OpenAI-whisper +This Firefox extension allows you to send audio from your browser to a server for transcribing the audio in real time. + +## Implementation Details + +### Capturing Audio +To capture the audio in the current tab, we used the chrome `tabCapture` API to obtain a `MediaStream` object of the current tab. + +### Getting Started +- Make sure the transcription server is running properly. To know more about how to start the server, see the [documentation here](https://github.com/collabora/whisper-live). +- Just click on the Firefox Extension which should show 2 options + - **Start Capture** : Starts capturing the audio in the current tab and sends the captured audio to the server for transcription. This also creates an element to show the transcriptions recieved from the server on the current tab. + - **Stop Capture** - Stops capturing the audio. + + +## Limitations +This extension requires an internet connection to stream audio and receive transcriptions. The accuracy of the transcriptions may vary depending on the audio quality and the performance of the server-side transcription service. The extension may consume additional system resources while running, especially when streaming audio. + diff --git a/Audio-Transcription-Firefox/background.js b/Audio-Transcription-Firefox/background.js new file mode 100644 index 0000000..9380328 --- /dev/null +++ b/Audio-Transcription-Firefox/background.js @@ -0,0 +1,18 @@ +browser.runtime.onMessage.addListener(function(request, sender, sendResponse) { + const { action, data } = request; + if (action === "transcript") { + browser.tabs.query({ active: true, currentWindow: true }) + .then((tabs) => { + const tabId = tabs[0].id; + browser.tabs.sendMessage(tabId, { action: "show_transcript", data }); + }) + .catch((error) => { + console.error("Error retrieving active tab:", error); + }); + } +}); + + + + + diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js new file mode 100644 index 0000000..9c2e518 --- /dev/null +++ b/Audio-Transcription-Firefox/content.js @@ -0,0 +1,283 @@ +console.log("Content script injected."); +let socket = null; +let isCapturing = false; +let mediaStream = null; +let audioContext = null; +let scriptProcessor = null; + +/** + * Resamples the audio data to a target sample rate of 16kHz. + * @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data. + * @param {number} [origSampleRate=44100] - The original sample rate of the audio data. + * @returns {Float32Array} The resampled audio data at 16kHz. + */ +function resampleTo16kHZ(audioData, origSampleRate = 44100) { + // Convert the audio data to a Float32Array + const data = new Float32Array(audioData); + + // Calculate the desired length of the resampled data + const targetLength = Math.round(data.length * (16000 / origSampleRate)); + + // Create a new Float32Array for the resampled data + const resampledData = new Float32Array(targetLength); + + // Calculate the spring factor and initialize the first and last values + const springFactor = (data.length - 1) / (targetLength - 1); + resampledData[0] = data[0]; + resampledData[targetLength - 1] = data[data.length - 1]; + + // Resample the audio data + for (let i = 1; i < targetLength - 1; i++) { + const index = i * springFactor; + const leftIndex = Math.floor(index).toFixed(); + const rightIndex = Math.ceil(index).toFixed(); + const fraction = index - leftIndex; + resampledData[i] = data[leftIndex] + (data[rightIndex] - data[leftIndex]) * fraction; + } + + // Return the resampled data + return resampledData; +} + +function startRecording() { + socket = new WebSocket("ws://localhost:9090/"); + socket.onopen = function(e) { + socket.send("handshake"); + }; + + socket.onmessage = (event) => { + // console.log(event.data); + const data = event.data; + browser.runtime.sendMessage({ action: "transcript", data }) + .catch(function(error) { + console.error("Error sending message:", error); + }); + }; + + // Access the audio stream from the current tab + navigator.mediaDevices.getUserMedia({ audio: true }) + .then(function(stream) { + // Create a new MediaRecorder instance + const audioDataCache = []; + audioContext = new AudioContext(); + mediaStream = audioContext.createMediaStreamSource(stream); + recorder = audioContext.createScriptProcessor(4096, 1, 1); + + recorder.onaudioprocess = async (event) => { + if (!audioContext || !isCapturing) return; + + const inputData = event.inputBuffer.getChannelData(0); + const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate); + + audioDataCache.push(inputData); + + // feed inputs and run + socket.send(audioData16kHz); + }; + + // Prevent page mute + mediaStream.connect(recorder); + recorder.connect(audioContext.destination); + }) +} + +var elem_container = null; +var elem_text = null; + +var segments = []; +var text_segments = []; + +function init_element() { + if (document.getElementById('transcription')) { + return; + } + + elem_container = document.createElement('div'); + elem_container.id = "transcription"; + elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;top:0px;position:absolute;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;'; + + for (var i = 0; i < 4; i++) { + elem_text = document.createElement('span'); + elem_text.style.cssText = 'position: absolute;padding-left:16px;padding-right:16px;'; + elem_text.id = "t" + i; + elem_container.appendChild(elem_text); + + if (i == 3) { + elem_text.style.top = "-1000px" + } + } + + document.body.appendChild(elem_container); + + let x = 0; + let y = 0; + + // Query the element + const ele = elem_container; + + // Handle the mousedown event + // that's triggered when user drags the element + const mouseDownHandler = function (e) { + // Get the current mouse position + x = e.clientX; + y = e.clientY; + + // Attach the listeners to `document` + document.addEventListener('mousemove', mouseMoveHandler); + document.addEventListener('mouseup', mouseUpHandler); + }; + + const mouseMoveHandler = function (e) { + // How far the mouse has been moved + const dx = e.clientX - x; + const dy = e.clientY - y; + + // Set the position of element + ele.style.top = `${ele.offsetTop + dy}px`; + ele.style.left = `${ele.offsetLeft + dx}px`; + + // Reassign the position of mouse + x = e.clientX; + y = e.clientY; + }; + + const mouseUpHandler = function () { + // Remove the handlers of `mousemove` and `mouseup` + document.removeEventListener('mousemove', mouseMoveHandler); + document.removeEventListener('mouseup', mouseUpHandler); + }; + + ele.addEventListener('mousedown', mouseDownHandler); +} + +function getStyle(el,styleProp) +{ + var x = document.getElementById(el); + if (x.currentStyle) + var y = x.currentStyle[styleProp]; + else if (window.getComputedStyle) + var y = document.defaultView.getComputedStyle(x,null).getPropertyValue(styleProp); + return y; +} + +function get_lines(elem, line_height) { + var divHeight = elem.offsetHeight; + var lines = divHeight / line_height; + + var original_text = elem.innerHTML; + + var words = original_text.split(' '); + var segments = []; + var current_lines = 1; + var segment = ''; + var segment_len = 0; + for (var i = 0; i < words.length; i++) + { + segment += words[i] + ' '; + elem.innerHTML = segment; + divHeight = elem.offsetHeight; + + if ((divHeight / line_height) > current_lines) { + var line_segment = segment.substring(segment_len, segment.length - 1 - words[i].length - 1); + segments.push(line_segment); + segment_len += line_segment.length + 1; + current_lines++; + } + } + + var line_segment = segment.substring(segment_len, segment.length - 1) + segments.push(line_segment); + + elem.innerHTML = original_text; + + return segments; + +} + +function remove_element() { + var elem = document.getElementById('transcription') + for (var i = 0; i < 4; i++) { + document.getElementById("t" + i).remove(); + } + elem.remove() +} + + +browser.runtime.onMessage.addListener((request, sender, sendResponse) => { + const { action, data } = request; + if (action === "startCapture") { + isCapturing = true; + startRecording(); + } else if (action === "stopCapture") { + + isCapturing = false; + if (socket) { + socket.close(); + socket = null; + } + + if (audioContext) { + audioContext.close(); + audioContext = null; + mediaStream = null; + recorder = null; + } + + + remove_element(); + + } else if (action === "show_transcript"){ + if (!isCapturing) return; + init_element(); + message = JSON.parse(data); + + var text = ''; + for (var i = 0; i < message.length; i++) { + text += message[i].text + ' '; + } + text = text.replace(/(\r\n|\n|\r)/gm, ""); + + var elem = document.getElementById('t3'); + elem.innerHTML = text; + + var line_height_style = getStyle('t3', 'line-height'); + var line_height = parseInt(line_height_style.substring(0, line_height_style.length - 2)); + var divHeight = elem.offsetHeight; + var lines = divHeight / line_height; + + text_segments = []; + text_segments = get_lines(elem, line_height); + + elem.innerHTML = ''; + + if (text_segments.length > 2) { + for (var i = 0; i < 3; i++) { + document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i]; + } + } else { + for (var i = 0; i < 3; i++) { + document.getElementById('t' + i).innerHTML = ''; + } + } + + if (text_segments.length <= 2) { + for (var i = 0; i < text_segments.length; i++) { + document.getElementById('t' + i).innerHTML = text_segments[i]; + } + } else { + for (var i = 0; i < 3; i++) { + document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i]; + } + } + + for (var i = 1; i < 3; i++) + { + var parent_elem = document.getElementById('t' + (i - 1)); + var elem = document.getElementById('t' + i); + elem.style.top = parent_elem.offsetHeight + parent_elem.offsetTop + 'px'; + } + } + sendResponse({}); +}); + + diff --git a/Audio-Transcription-Firefox/icon128.png b/Audio-Transcription-Firefox/icon128.png new file mode 100644 index 0000000000000000000000000000000000000000..3234deb35dd9640376ebf06daef1726cd8e4d879 GIT binary patch literal 3054 zcmV3zgkM1U1ub26W7G0DBDkB3H*e^(ofnTg18pMbxjbW zT}lE7-~sjp2|kOyTocz7L4;;02q1tzCq6FK*sW(7E-niqT(yz_(&FbC_*{r^-jqjh z4FKPez|S%^xk^(Kp_S$)UP|!a5g&^Tejx&#FG>vZ^#C%npXUV68bMxgQKw>XHNc0< ze^=JCDrAu*2i0GC)tm6uDA1d#E8t-l^# zU^dDFTwvYQMlN*q0c5*yh4oS^nZyE;zyol{uYPhi90KN8AGMRYFac&+AGH%7b5}0* zSY&5MYY+P+OS-0eHbjePtR1Xs{k?D-9+96M&`8!UU+f3}$)rf%dR7 zbbHJhyfxv=ih9OZ#bJ1xYnJ{vG1KHPp?&4;X^_fgSSV8K1LFC(h<6Na%|22 zxikbAxp~+b{`9c3Z~2fvK&otybe#1QE6(G_s`KdOJCy)^|G$np`dHRG(+0gC8DJRS zpZdnx68b5W00Ho0H=Z^+$SA>}%Rqzxco;*L0^?Xabt^Ryvst>)%VK}+QKj>m6S&T|L~uFVg>jOP$=ymG67+?N8`c03Lea!)LXCfACr>Rm&3W$Md6zP!#wOWcxbh zFMx+Jq^VfYcR46Lj}8$YzVhGL-pOEGHi?FY$p?`XCKLtMj@>x!lQ6J2Rq$as7=uZY zfRt45@xIW>+Np0+KOY8#OUE#QB|J}5-^7c{tSN2|18s|%+;oQyiUZ)dKLvcmK599X zB-ErYcwZCXcN^K)zOJ_ki7&8>%Uis(Ob-TXdX_2>0FL`pz(?{#Er*h_ev0eOI!r;= zfnlf$Fa@@W1Q`41IO%23jmtV7G&+dlHeCp6X8ku$*H3NcM_u z#m0eeiUibk4S5(uI9CD&X*`+Nn`u%qThsB)H25Y+5M?!Sf~ncO4FVVd3raN+O8`n= zef9M(N&$Xa5@10GCF3tb9{il<1tJLm*Pz;Y!r}}YreZNIP3e|VNdRk6t%k5pA@7!S zI3G}5J{PRd+(Uq<46&c9vq4~S+tm4QN}rrSuA+59d43q{T>_|D1uWl|mAgTSm=y_B zjRK1?in9K&E>yj8c-*@Lz=VwIU~Q5tNicM4RIM&QX@^c!u?b`CQ3p%OILl4~)%cJ7 z3^s@=Heswi>R_>H!A3Dn0!mI~OGs43CXBVGqi9zp7iC$ee$y$6r16r$<{ zuwLJW#OMBNEz&VE)KU+LplU|b?f6?70De^(huS>grP>se3&tT1913DE@)HnuS_~nm&)9ofU{kSspzk{R_1m8cE#bOZ*bn?$X_MfqHcw+y>7dT4~ zhcLRXW9;kzJa^ynxU#EtJDxN8@iC?M)e%%2)Gg{*Gr0cKyU7Ib-QKeg{Q+Pgq)b#O z2_*(dWFoma^tTtf$}0*9)?d17be$f?MUC%xUy$*MFpTZNt<>+Za~Rc0vd7)oyZKq< zJ46xy|8Vojx-;_05%LR;ikEXc13pN)gZ9tf5mq1Ng;YskLgBrh5nhfO-*HqfjA`q@ z6K!t*V;KAAGtR_&FFJ$Y?WgBQZ+;#-x6${Mur%>KIOS9Ic+()x<5KBpsg{4H<;Wj? zrhRe;bb@!rr83zR9J`z5v{5_@x{3`~{RlwWd6Vcug&PbGK@NKi9F|Ny9;QeVKqp6k zKVxJXI18OnE1<-40x{qp2-e#=4O|2XY(6+EoVeco)xQ(-pu{W>H^AY8+XZiYh?63Y ziPK$on3rrz%v6$KPj$fG2{v^sTM=B;F>yANpd+zTUC`oDTnYwk%f=AuK(iu&O@;w% zfDvV3Yh)SNue92Uw{D@cfoN*@2P2Eps(3H7>GvGaGodukOba&3A|IlbzuL#eBN00I zkA{(KmEwgkNE=Qii4qtEO6NQiW>`JW2hjC|slPDe_ht^4gwL-J~sJ10$8Q=X!?$^Q`w9WAlx^=@unm4i=(KuGF;k* zJQT4iyL8|Q{H`93YSTCq86gi_(EA-RYYhRS29M6xkUYCD8uV3vBSzJ~GA^7yxf=tq z%eVo$f8fJ-_FL65@1O+x?QBiliYgy{;@fb<9>Y;@VFEA#oEEF0w$fq(Faew;)ksz<7FZv(lZ7w= zuCYF9C)aifAYwtANl-8FCFWhN;47?`+Q=@}Zun0{HSm@AXs~`NEiGaF&j-Khoea9n z`l-BhVeIz)Eqj6bSbvlUfShMNRaW+3yN3XRAx(!HLsgWf#MVjj00C3lth35UJI=F5 z5m2$k6fiGuJbxpZO87k@;E?V+J?T$^lO==yVR{k%*08;8r@|KJit^fc407*qoM6N<$f@U$DaR2}S literal 0 HcmV?d00001 diff --git a/Audio-Transcription-Firefox/manifest.json b/Audio-Transcription-Firefox/manifest.json new file mode 100644 index 0000000..e3cd717 --- /dev/null +++ b/Audio-Transcription-Firefox/manifest.json @@ -0,0 +1,27 @@ +{ + "manifest_version": 2, + "name": "Audio Recorder", + "version": "1.0", + "description": "Record audio from any webpage.", + "permissions": [ + "activeTab", + "" + ], + "background": { + "scripts": ["background.js"], + "persistent": false + }, + "browser_action": { + "default_popup": "popup.html", + "default_icon": "icon128.png" + }, + "icons": { + "128":"icon128.png" + }, + "content_scripts": [ + { + "matches": [""], + "js": ["content.js"] + } + ] + } \ No newline at end of file diff --git a/Audio-Transcription-Firefox/popup.html b/Audio-Transcription-Firefox/popup.html new file mode 100644 index 0000000..b961819 --- /dev/null +++ b/Audio-Transcription-Firefox/popup.html @@ -0,0 +1,15 @@ + + + + Audio Capture + + + + +

Audio Transcription

+
+
Start Capture
+
Stop Capture
+
+ + \ No newline at end of file diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js new file mode 100644 index 0000000..bc37fc4 --- /dev/null +++ b/Audio-Transcription-Firefox/popup.js @@ -0,0 +1,38 @@ +document.addEventListener("DOMContentLoaded", function() { + var startButton = document.getElementById("startCapture"); + var stopButton = document.getElementById("stopCapture"); + + startButton.addEventListener("click", function() { + browser.tabs.query({ active: true, currentWindow: true }) + .then(function(tabs) { + browser.tabs.sendMessage(tabs[0].id, { action: "startCapture" }); + toggleCaptureButtons(true); + }) + .catch(function(error) { + console.error("Error sending startCapture message:", error); + }); + }); + + stopButton.addEventListener("click", function() { + browser.tabs.query({ active: true, currentWindow: true }) + .then(function(tabs) { + browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture" }) + .then(function(response) { + console.log(response); + toggleCaptureButtons(false); + }) + .catch(function(error) { + console.error("Error sending stopCapture message:", error); + }); + }) + .catch(function(error) { + console.error("Error querying active tab:", error); + }); + }); + + // Function to toggle the capture buttons + function toggleCaptureButtons(isCapturing) { + startButton.disabled = isCapturing; + stopButton.disabled = !isCapturing; + } +}); \ No newline at end of file diff --git a/Audio-Transcription-Firefox/style.css b/Audio-Transcription-Firefox/style.css new file mode 100644 index 0000000..c1c56ca --- /dev/null +++ b/Audio-Transcription-Firefox/style.css @@ -0,0 +1,103 @@ +.header { + display: flex; + align-items: center; + padding-bottom: 15px; + padding-left: 20px; + border-bottom: 2px solid darkred; +} + +.header-title { + padding: 0 5px; +} + +h1 { + font-size: 36px; +} + +img { + height: 64px; + margin: 0 20px 0 0; +} + +h2 { + font-size: 26px; +} + +label { + font-size: 16px; +} + +.inner { + margin-left: 40px; +} + +.options-list { + padding: 0; + list-style: none; +} + +.options-list li { + padding: 10px; +} + +.time { + font-size: 16px; +} + +.limit { + display: inline-block; + margin: 0; + font-size: 12px; +} + +.radioChoice { + margin-left: 15px; +} + +.button-container { + display: flex; + justify-content: space-between; + padding: 10px; +} + +.button { + padding: 10px; + border: 2px solid darkred; + font-size: 16px; + font-weight: bold; + cursor: pointer; + white-space: nowrap; + width: 150px; + border-radius: 5px; +} + +.disabled { + opacity: 0.6; + cursor: not-allowed; +} + +.button:hover:not(:disabled) { + color: red; + background-color: darkred; +} + +#save { + font-size: 16px; + margin-left: 50px; +} + +#status { + color: red; + margin-top: 8px; + margin-left: 50px; + font-size: 14px; +} + +#qualityLi { + display: none; +} + +#maxTime { + width: 30px; + text-align: center; +}