From ad0fb239366af889e54cb95ec29c627758057853 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 14 Jul 2025 09:43:52 +0530 Subject: [PATCH] Add download srt file option firefox extension Signed-off-by: makaveli10 --- Audio-Transcription-Firefox/README.md | 1 + .../audiopreprocessor.js | 70 +++++ Audio-Transcription-Firefox/content.js | 285 ++++++++++-------- Audio-Transcription-Firefox/manifest.json | 3 + Audio-Transcription-Firefox/popup.html | 4 + Audio-Transcription-Firefox/popup.js | 16 +- 6 files changed, 258 insertions(+), 121 deletions(-) create mode 100644 Audio-Transcription-Firefox/audiopreprocessor.js diff --git a/Audio-Transcription-Firefox/README.md b/Audio-Transcription-Firefox/README.md index 5e4669e..f9ac237 100644 --- a/Audio-Transcription-Firefox/README.md +++ b/Audio-Transcription-Firefox/README.md @@ -25,6 +25,7 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to When using the Audio Transcription extension, you have the following options: - **Use Collabora Server**: We provide a demo server which runs the whisper small model. - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper. + - **Download SRT file at Stop Capture**: Select if you want to download the srt file for the session at stop capture. - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English. - **Model Size**: Select the whisper model size to run the server with. diff --git a/Audio-Transcription-Firefox/audiopreprocessor.js b/Audio-Transcription-Firefox/audiopreprocessor.js new file mode 100644 index 0000000..2a05089 --- /dev/null +++ b/Audio-Transcription-Firefox/audiopreprocessor.js @@ -0,0 +1,70 @@ +class AudioPreProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.sampleRate = sampleRate || 48000; + this.targetSampleRate = 16000; + this.inputSamplesNeeded = this.sampleRate * 0.5; + this.inputBuffer = new Float32Array(this.inputSamplesNeeded); + this.inputWriteOffset = 0; + } + + process(inputs, outputs) { + const input = inputs[0]; + const output = outputs[0]; + if (!input || input.length === 0) { + return true; + } + for (let channel = 0; channel < Math.min(input.length, output.length); channel++) { + if (input[channel] && output[channel]) { + output[channel].set(input[channel]); + } + } + + let monoInput; + if (input.length === 1) { + monoInput = input[0]; + } else if (input.length > 1) { + monoInput = new Float32Array(input[0].length); + for (let channel = 0; channel < input.length; channel++) { + monoInput.set(input[channel], 0); + } + } + + if (!monoInput) { + return true; + } + + let inputOffset = 0; + while (inputOffset < monoInput.length) { + const remainingBuffer = this.inputSamplesNeeded - this.inputWriteOffset; + const toCopy = Math.min(remainingBuffer, monoInput.length - inputOffset); + this.inputBuffer.set(monoInput.subarray(inputOffset, inputOffset + toCopy), this.inputWriteOffset); + + this.inputWriteOffset += toCopy; + inputOffset += toCopy; + + if (this.inputWriteOffset === this.inputSamplesNeeded) { + const downsampled = this.downsampleTo16kHz(this.inputBuffer); + this.port.postMessage(downsampled); + + this.inputWriteOffset = 0; + } + } + + return true; + } + + downsampleTo16kHz(inputBuffer) { + const ratio = this.sampleRate / this.targetSampleRate; + const length = Math.floor(inputBuffer.length / ratio); + const result = new Float32Array(length); + for (let i = 0; i < length; i++) { + const idx = Math.floor(i * ratio); + result[i] = inputBuffer[idx]; + } + return result; + } + } + + registerProcessor('audiopreprocessor', AudioPreProcessor); + \ No newline at end of file diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index 6af6861..6cd4e7c 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -1,144 +1,162 @@ let socket = null; let isCapturing = false; -let mediaStream = null; let audioContext = null; -let scriptProcessor = null; let language = null; - let isPaused = false; +let preNode = null; +let allSegments = []; +let lastIncompleteSegment = null; -const mediaElements = document.querySelectorAll('video, audio'); -mediaElements.forEach((mediaElement) => { - mediaElement.addEventListener('play', handlePlaybackStateChange); - mediaElement.addEventListener('pause', handlePlaybackStateChange); -}); - - -function handlePlaybackStateChange(event) { - isPaused = event.target.paused; +function formatTime(seconds) { + const date = new Date(seconds * 1000); + const hh = String(date.getUTCHours()).padStart(2, '0'); + const mm = String(date.getUTCMinutes()).padStart(2, '0'); + const ss = String(date.getUTCSeconds()).padStart(2, '0'); + const mmm = String(date.getUTCMilliseconds()).padStart(3, '0'); + return `${hh}:${mm}:${ss},${mmm}`; } +function generateSRT() { + return allSegments + .map((seg, i) => { + const start = formatTime(seg.start); + const end = formatTime(seg.end); + const text = seg.text.trim().replace(/[\r\n]+/g, ' '); + return `${i + 1}\n${start} --> ${end}\n${text}`; + }) + .join('\n\n'); +} + +function downloadSRT() { + const srtBlob = new Blob([generateSRT()], { type: 'text/srt;charset=utf-8' }); + const url = URL.createObjectURL(srtBlob); + const a = document.createElement('a'); + a.href = url; + a.download = 'captions.srt'; + a.style.display = 'none'; + document.body.appendChild(a); + a.click(); + URL.revokeObjectURL(url); + document.body.removeChild(a); +} + + function generateUUID() { let dt = new Date().getTime(); - const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { + return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => { const r = (dt + Math.random() * 16) % 16 | 0; dt = Math.floor(dt / 16); return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16); }); - return uuid; } -/** - * Resamples the audio data to a target sample rate of 16kHz. - * @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data. - * @param {number} [origSampleRate=44100] - The original sample rate of the audio data. - * @returns {Float32Array} The resampled audio data at 16kHz. - */ -function resampleTo16kHZ(audioData, origSampleRate = 44100) { - // Convert the audio data to a Float32Array - const data = new Float32Array(audioData); +document.querySelectorAll('video, audio').forEach(el => { + el.addEventListener('play', () => { isPaused = false; }); + el.addEventListener('pause', () => { isPaused = true; }); +}); - // Calculate the desired length of the resampled data - const targetLength = Math.round(data.length * (16000 / origSampleRate)); - // Create a new Float32Array for the resampled data - const resampledData = new Float32Array(targetLength); +function setupMessageHandler() { + if (preNode) { + preNode.port.onmessage = e => { + const audio16k = e.data; + if (isCapturing && socket && socket.readyState === WebSocket.OPEN && !isPaused) { + socket.send(audio16k); + } + }; + } +} - // Calculate the spring factor and initialize the first and last values - const springFactor = (data.length - 1) / (targetLength - 1); - resampledData[0] = data[0]; - resampledData[targetLength - 1] = data[data.length - 1]; - // Resample the audio data - for (let i = 1; i < targetLength - 1; i++) { - const index = i * springFactor; - const leftIndex = Math.floor(index).toFixed(); - const rightIndex = Math.ceil(index).toFixed(); - const fraction = index - leftIndex; - resampledData[i] = data[leftIndex] + (data[rightIndex] - data[leftIndex]) * fraction; +const WORKLET_URL = browser.runtime.getURL('audiopreprocessor.js'); + +async function initAudioWorklet() { + if (audioContext && preNode) { + setupMessageHandler(); + return; + } + audioContext = new AudioContext(); + await audioContext.audioWorklet.addModule(WORKLET_URL); + + preNode = new AudioWorkletNode(audioContext, 'audiopreprocessor'); + document.querySelectorAll('audio, video').forEach(el => { + let src; + try { + src = audioContext.createMediaElementSource(el); + } catch(e) { + console.warn('Could not create MediaElementSource for', el, e); + return; + } + src.connect(preNode); + src.connect(audioContext.destination); + }); + + preNode.connect(audioContext.destination); + + setupMessageHandler(); +} + +async function startRecording(data) { + if (!audioContext) { + await initAudioWorklet(); } - // Return the resampled data - return resampledData; + const uid = generateUUID(); + socket = new WebSocket(`ws://${data.host}:${data.port}/`); + language = data.language; + + socket.onopen = () => { + socket.send(JSON.stringify({ + uid, + language: data.language, + task: data.task, + model: data.modelSize, + use_vad: data.useVad + })); + }; + + let serverReady = false; + socket.onmessage = async event => { + const msg = JSON.parse(event.data); + if (msg.uid !== uid) return; + + if (msg.status === 'WAIT') { + await browser.runtime.sendMessage({ action: 'showPopup', data: msg.message }); + return; + } + if (!serverReady && msg.message === 'SERVER_READY') { + serverReady = true; + return; + } + if (!language && msg.language) { + language = msg.language; + await browser.runtime.sendMessage({ action: 'updateSelectedLanguage', data: language }); + return; + } + if (msg.message === 'DISCONNECT') { + await browser.runtime.sendMessage({ action: 'toggleCaptureButtons' }); + return; + } + if (msg.segments) { + await browser.runtime.sendMessage({ action: 'transcript', data: {data: event.data, saveCaption: data.saveCaption} }); + } + }; + + isCapturing = true; } -function startRecording(data) { - socket = new WebSocket(`ws://${data.host}:${data.port}/`); - language = data.language; +function stopRecording() { + isCapturing = false; + if (socket) { + socket.close(); + socket = null; + } - const uuid = generateUUID(); - socket.onopen = function(e) { - socket.send( - JSON.stringify({ - uid: uuid, - language: data.language, - task: data.task, - model: data.modelSize, - use_vad: data.useVad - }) - ); - }; - - let isServerReady = false; - socket.onmessage = async (event) => { - const data = JSON.parse(event.data); - if (data["uid"] !== uuid) - return; - - if (data["status"] === "WAIT"){ - await browser.runtime.sendMessage({ action: "showPopup", data: data["message"] }) - return; - } - - if (!isServerReady && data["message"] === "SERVER_READY"){ - isServerReady = true; - return; - } - - if (language === null ){ - language = data["language"]; - await browser.runtime.sendMessage({ action: "updateSelectedLanguage", data: language }) - return - } - - if (data["message"] === "DISCONNECT"){ - await browser.runtime.sendMessage({ action: "toggleCaptureButtons", data: false }) - return - } - - await browser.runtime.sendMessage({ action: "transcript", data: event.data }) - .catch(function(error) { - console.error("Error sending message:", error); - }); - }; - - // Access the audio stream from the current tab - navigator.mediaDevices.getUserMedia({ audio: true }) - .then(function(stream) { - // Create a new MediaRecorder instance - const audioDataCache = []; - audioContext = new AudioContext(); - mediaStream = audioContext.createMediaStreamSource(stream); - recorder = audioContext.createScriptProcessor(4096, 1, 1); - - recorder.onaudioprocess = async (event) => { - if (!audioContext || !isCapturing || !isServerReady || isPaused) return; - - const inputData = event.inputBuffer.getChannelData(0); - const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate); - - audioDataCache.push(inputData); - - socket.send(audioData16kHz); - }; - - // Prevent page mute - mediaStream.connect(recorder); - recorder.connect(audioContext.destination); - }) + remove_element(); } + var elem_container = null; var elem_text = null; @@ -308,6 +326,8 @@ function remove_element() { browser.runtime.onMessage.addListener((request, sender, sendResponse) => { const { action, data } = request; + const saveCaption = data.saveCaption || false; + if (action === "startCapture") { isCapturing = true; startRecording(data); @@ -318,12 +338,20 @@ browser.runtime.onMessage.addListener((request, sender, sendResponse) => { socket.close(); socket = null; } - - if (audioContext) { - audioContext.close(); - audioContext = null; - mediaStream = null; - recorder = null; + + + if (saveCaption === true) { + if (lastIncompleteSegment && lastIncompleteSegment.text && lastIncompleteSegment.text.trim() !== "") { + if (allSegments.length === 0 || parseFloat(lastIncompleteSegment.start) >= parseFloat(allSegments[allSegments.length - 1].end)) { + allSegments.push({ + start: lastIncompleteSegment.start, + end: lastIncompleteSegment.end, + text: lastIncompleteSegment.text + }); + } + } + + downloadSRT(); } remove_element(); @@ -337,8 +365,25 @@ browser.runtime.onMessage.addListener((request, sender, sendResponse) => { } else if (action === "show_transcript"){ if (!isCapturing) return; init_element(); - message = JSON.parse(data); + message = JSON.parse(data.data); message = message["segments"]; + + if (saveCaption === true) { + message.forEach(seg => { + if (seg.completed === true && + (allSegments.length === 0 || parseFloat(seg.start) >= parseFloat(allSegments[allSegments.length - 1].end))) { + allSegments.push({ + start: seg.start, + end: seg.end, + text: seg.text + }); + + lastIncompleteSegment = null; + } else if (seg.completed !== true) { + lastIncompleteSegment = seg; + } + }); + } var text = ''; for (var i = 0; i < message.length; i++) { diff --git a/Audio-Transcription-Firefox/manifest.json b/Audio-Transcription-Firefox/manifest.json index 9c0114e..1b83330 100644 --- a/Audio-Transcription-Firefox/manifest.json +++ b/Audio-Transcription-Firefox/manifest.json @@ -8,6 +8,9 @@ "activeTab", "" ], + "web_accessible_resources": [ + "audiopreprocessor.js" + ], "background": { "scripts": ["background.js"], "persistent": false diff --git a/Audio-Transcription-Firefox/popup.html b/Audio-Transcription-Firefox/popup.html index 8b2aac5..87fc131 100644 --- a/Audio-Transcription-Firefox/popup.html +++ b/Audio-Transcription-Firefox/popup.html @@ -19,6 +19,10 @@ +
+ + +
diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js index 61c37a5..b11adc3 100644 --- a/Audio-Transcription-Firefox/popup.js +++ b/Audio-Transcription-Firefox/popup.js @@ -4,6 +4,7 @@ document.addEventListener("DOMContentLoaded", function() { const useServerCheckbox = document.getElementById("useServerCheckbox"); const useVadCheckbox = document.getElementById("useVadCheckbox"); + const saveCaptionCheckbox = document.getElementById("saveCaptionCheckbox"); const languageDropdown = document.getElementById('languageDropdown'); const taskDropdown = document.getElementById('taskDropdown'); const modelSizeDropdown = document.getElementById('modelSizeDropdown'); @@ -41,6 +42,12 @@ document.addEventListener("DOMContentLoaded", function() { } }); + browser.storage.local.get("saveCaptionState", ({ saveCaptionState }) => { + if (saveCaptionState !== undefined) { + saveCaptionCheckbox.checked = saveCaptionState; + } + }); + browser.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => { if (storedLanguage !== undefined) { languageDropdown.value = storedLanguage; @@ -85,6 +92,7 @@ document.addEventListener("DOMContentLoaded", function() { task: selectedTask, modelSize: selectedModelSize, useVad: useVadCheckbox.checked, + saveCaption: saveCaptionCheckbox.checked, } }); toggleCaptureButtons(true); @@ -101,7 +109,7 @@ document.addEventListener("DOMContentLoaded", function() { stopButton.addEventListener("click", function() { browser.tabs.query({ active: true, currentWindow: true }) .then(function(tabs) { - browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture" }) + browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture", data: {saveCaption: saveCaptionCheckbox.checked, } }) .then(function(response) { toggleCaptureButtons(false); browser.storage.local.set({ capturingState: { isCapturing: false } }) @@ -124,6 +132,7 @@ document.addEventListener("DOMContentLoaded", function() { stopButton.disabled = !isCapturing; useServerCheckbox.disabled = isCapturing; useVadCheckbox.disabled = isCapturing; + saveCaptionCheckbox.disabled = isCapturing; modelSizeDropdown.disabled = isCapturing; languageDropdown.disabled = isCapturing; taskDropdown.disabled = isCapturing; @@ -142,6 +151,11 @@ document.addEventListener("DOMContentLoaded", function() { browser.storage.local.set({ useVadState }); }); + saveCaptionCheckbox.addEventListener("change", () => { + const saveCaptionState = saveCaptionCheckbox.checked; + browser.storage.local.set({ saveCaptionState }); + }); + languageDropdown.addEventListener('change', function() { if (languageDropdown.value === "") { selectedLanguage = null;