From 914281f449a1dfcc00f441f4a208dd17435c49b5 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 14 Jul 2025 09:43:39 +0530 Subject: [PATCH 1/2] Add download srt file option chrome extension Signed-off-by: makaveli10 --- Audio-Transcription-Chrome/README.md | 1 + .../audiopreprocessor.js | 77 ++++++++ Audio-Transcription-Chrome/background.js | 9 +- Audio-Transcription-Chrome/content.js | 172 +++++++++++++----- Audio-Transcription-Chrome/manifest.json | 10 +- Audio-Transcription-Chrome/options.js | 155 +++++++++------- Audio-Transcription-Chrome/popup.html | 4 + Audio-Transcription-Chrome/popup.js | 20 +- 8 files changed, 330 insertions(+), 118 deletions(-) create mode 100644 Audio-Transcription-Chrome/audiopreprocessor.js diff --git a/Audio-Transcription-Chrome/README.md b/Audio-Transcription-Chrome/README.md index ec00040..b75eda7 100644 --- a/Audio-Transcription-Chrome/README.md +++ b/Audio-Transcription-Chrome/README.md @@ -27,6 +27,7 @@ To capture the audio in the current tab, we used the chrome `tabCapture` API to When using the Audio Transcription extension, you have the following options: - **Use Collabora Server**: We provide a demo server which runs the whisper small model. - **Language**: Select the target language for transcription or translation. You can choose from a variety of languages supported by OpenAI-whisper. + - **Download SRT file at Stop Capture**: Select if you want to download the srt file for the session at stop capture. - **Task:** Choose the specific task to perform on the audio. You can select either "transcribe" for transcription or "translate" to translate the audio to English. - **Model Size**: Select the whisper model size to run the server with. diff --git a/Audio-Transcription-Chrome/audiopreprocessor.js b/Audio-Transcription-Chrome/audiopreprocessor.js new file mode 100644 index 0000000..c549446 --- /dev/null +++ b/Audio-Transcription-Chrome/audiopreprocessor.js @@ -0,0 +1,77 @@ +class AudioPreProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.sampleRate = sampleRate || 48000; + this.targetSampleRate = 16000; + this.inputSamplesNeeded = this.sampleRate * 0.5; // 0.5s + this.inputBuffer = new Float32Array(this.inputSamplesNeeded); + this.inputWriteOffset = 0; + this.processCount = 0; + this.audioDetectedCount = 0; + } + + process(inputs, outputs) { + this.processCount++; + + const input = inputs[0]; + const output = outputs[0]; + + if (!input || input.length === 0) { + return true; + } + + for (let channel = 0; channel < Math.min(input.length, output.length); channel++) { + if (input[channel] && output[channel]) { + output[channel].set(input[channel]); + } + } + + let monoInput; + if (input.length === 1) { + monoInput = input[0]; + } else if (input.length >= 2) { + monoInput = new Float32Array(input[0].length); + for (let i = 0; i < input[0].length; i++) { + monoInput[i] = (input[0][i] + (input[1] ? input[1][i] : 0)) * 0.5; + } + } else { + return true; + } + + if (!monoInput || monoInput.length === 0) { + return true; + } + + let inputOffset = 0; + while (inputOffset < monoInput.length) { + const remainingBuffer = this.inputSamplesNeeded - this.inputWriteOffset; + const toCopy = Math.min(remainingBuffer, monoInput.length - inputOffset); + this.inputBuffer.set(monoInput.subarray(inputOffset, inputOffset + toCopy), this.inputWriteOffset); + + this.inputWriteOffset += toCopy; + inputOffset += toCopy; + + if (this.inputWriteOffset === this.inputSamplesNeeded) { + const downsampled = this.downsampleTo16kHz(this.inputBuffer); + this.port.postMessage(downsampled); + + this.inputWriteOffset = 0; + } + } + + return true; + } + + downsampleTo16kHz(inputBuffer) { + const ratio = this.sampleRate / this.targetSampleRate; + const length = Math.floor(inputBuffer.length / ratio); + const result = new Float32Array(length); + for (let i = 0; i < length; i++) { + const idx = Math.floor(i * ratio); + result[i] = inputBuffer[idx]; + } + return result; + } +} + +registerProcessor('audiopreprocessor', AudioPreProcessor); \ No newline at end of file diff --git a/Audio-Transcription-Chrome/background.js b/Audio-Transcription-Chrome/background.js index 52ce4a7..c3cc637 100644 --- a/Audio-Transcription-Chrome/background.js +++ b/Audio-Transcription-Chrome/background.js @@ -159,6 +159,7 @@ async function startCapture(options) { task: options.task, modelSize: options.modelSize, useVad: options.useVad, + saveCaptions: options.saveCaptions, }, }); } else { @@ -174,14 +175,14 @@ async function startCapture(options) { * Stops the capture process and performs cleanup. * @returns {Promise} - A Promise that resolves when the capture process is stopped successfully. */ -async function stopCapture() { +async function stopCapture(options) { const optionTabId = await getLocalStorageValue("optionTabId"); const currentTabId = await getLocalStorageValue("currentTabId"); if (optionTabId) { res = await sendMessageToTab(currentTabId, { type: "STOP", - data: { currentTabId: currentTabId }, + data: { currentTabId: currentTabId, saveCaptions: options.saveCaptions }, }); await removeChromeTab(optionTabId); } @@ -196,7 +197,7 @@ chrome.runtime.onMessage.addListener(async (message) => { if (message.action === "startCapture") { startCapture(message); } else if (message.action === "stopCapture") { - stopCapture(); + stopCapture(message); } else if (message.action === "updateSelectedLanguage") { const detectedLanguage = message.detectedLanguage; chrome.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage }); @@ -204,7 +205,7 @@ chrome.runtime.onMessage.addListener(async (message) => { } else if (message.action === "toggleCaptureButtons") { chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false }); chrome.storage.local.set({ capturingState: { isCapturing: false } }) - stopCapture(); + stopCapture({saveCaptions: message.saveCaptions}); } }); diff --git a/Audio-Transcription-Chrome/content.js b/Audio-Transcription-Chrome/content.js index c22376b..00bed88 100644 --- a/Audio-Transcription-Chrome/content.js +++ b/Audio-Transcription-Chrome/content.js @@ -1,10 +1,45 @@ - - var elem_container = null; var elem_text = null; var segments = []; var text_segments = []; +var allSegments = []; +var lastIncompleteSegment = null; + +function formatTime(seconds) { + const date = new Date(seconds * 1000); + const hh = String(date.getUTCHours()).padStart(2, '0'); + const mm = String(date.getUTCMinutes()).padStart(2, '0'); + const ss = String(date.getUTCSeconds()).padStart(2, '0'); + const mmm = String(date.getUTCMilliseconds()).padStart(3, '0'); + return `${hh}:${mm}:${ss},${mmm}`; +} + +function generateSRT() { + return allSegments + .map((seg, i) => { + const start = formatTime(seg.start); + const end = formatTime(seg.end); + const text = seg.text.trim().replace(/[\r\n]+/g, ' '); + return `${i + 1}\n${start} --> ${end}\n${text}`; + }) + .join('\n\n'); +} + +function downloadSRT() { + console.log("downloadSRT called"); + console.log("Total segments for SRT:", allSegments.length); + const srtBlob = new Blob([generateSRT()], { type: 'text/srt;charset=utf-8' }); + const url = URL.createObjectURL(srtBlob); + const a = document.createElement('a'); + a.href = url; + a.download = 'captions.srt'; + a.style.display = 'none'; + document.body.appendChild(a); + a.click(); + URL.revokeObjectURL(url); + document.body.removeChild(a); +} function initPopupElement() { if (document.getElementById('popupElement')) { @@ -32,7 +67,7 @@ function initPopupElement() { closePopupButton.style.cursor = 'pointer'; closePopupButton.addEventListener('click', async () => { popupContainer.style.display = 'none'; - await browser.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false }); + await chrome.runtime.sendMessage({ action: 'toggleCaptureButtons', data: false }); }); buttonContainer.appendChild(closePopupButton); popupContainer.appendChild(buttonContainer); @@ -169,8 +204,25 @@ function remove_element() { chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { const { type, data } = request; - - if (type === "STOP") { + const saveCaptions = data.saveCaptions; + + if (type === "STOP") { + if (saveCaptions === true) { + // If there is a last incomplete segment, push it to allSegments + if (lastIncompleteSegment && lastIncompleteSegment.text && lastIncompleteSegment.text.trim() !== "") { + // Apply same Python logic: check if transcript is empty OR start >= last end + if (allSegments.length === 0 || parseFloat(lastIncompleteSegment.start) >= parseFloat(allSegments[allSegments.length - 1].end)) { + allSegments.push({ + start: lastIncompleteSegment.start, + end: lastIncompleteSegment.end, + text: lastIncompleteSegment.text + }); + console.log("Added final incomplete segment"); + } + } + + downloadSRT(); + } remove_element(); sendResponse({data: "STOPPED"}); return true; @@ -184,53 +236,77 @@ chrome.runtime.onMessage.addListener((request, sender, sendResponse) => { init_element(); - message = JSON.parse(data); - message = message["segments"]; - - var text = ''; - for (var i = 0; i < message.length; i++) { - text += message[i].text + ' '; - } - text = text.replace(/(\r\n|\n|\r)/gm, ""); - - var elem = document.getElementById('t3'); - elem.innerHTML = text; - - var line_height_style = getStyle('t3', 'line-height'); - var line_height = parseInt(line_height_style.substring(0, line_height_style.length - 2)); - var divHeight = elem.offsetHeight; - var lines = divHeight / line_height; - - text_segments = []; - text_segments = get_lines(elem, line_height); - - elem.innerHTML = ''; - - if (text_segments.length > 2) { - for (var i = 0; i < 3; i++) { - document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i]; + try { + const message = JSON.parse(data.data); + const segments = message["segments"]; + + if (saveCaptions === true) { + segments.forEach(seg => { + if (seg.completed === true && + (allSegments.length === 0 || parseFloat(seg.start) >= parseFloat(allSegments[allSegments.length - 1].end))) { + allSegments.push({ + start: seg.start, + end: seg.end, + text: seg.text + }); + + lastIncompleteSegment = null; + } else if (seg.completed !== true) { + lastIncompleteSegment = seg; + } + }); } - } else { - for (var i = 0; i < 3; i++) { - document.getElementById('t' + i).innerHTML = ''; + var text = ''; + for (var i = 0; i < segments.length; i++) { + text += segments[i].text + ' '; } - } + text = text.replace(/(\r\n|\n|\r)/gm, ""); + + var elem = document.getElementById('t3'); + if (elem) { + elem.innerHTML = text; - if (text_segments.length <= 2) { - for (var i = 0; i < text_segments.length; i++) { - document.getElementById('t' + i).innerHTML = text_segments[i]; - } - } else { - for (var i = 0; i < 3; i++) { - document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i]; - } - } + var line_height_style = getStyle('t3', 'line-height'); + var line_height = parseInt(line_height_style.substring(0, line_height_style.length - 2)); + var divHeight = elem.offsetHeight; + var lines = divHeight / line_height; - for (var i = 1; i < 3; i++) - { - var parent_elem = document.getElementById('t' + (i - 1)); - var elem = document.getElementById('t' + i); - elem.style.top = parent_elem.offsetHeight + parent_elem.offsetTop + 'px'; + text_segments = []; + text_segments = get_lines(elem, line_height); + + elem.innerHTML = ''; + + if (text_segments.length > 2) { + for (var i = 0; i < 3; i++) { + document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i]; + } + } else { + for (var i = 0; i < 3; i++) { + document.getElementById('t' + i).innerHTML = ''; + } + } + + if (text_segments.length <= 2) { + for (var i = 0; i < text_segments.length; i++) { + document.getElementById('t' + i).innerHTML = text_segments[i]; + } + } else { + for (var i = 0; i < 3; i++) { + document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i]; + } + } + + for (var i = 1; i < 3; i++) + { + var parent_elem = document.getElementById('t' + (i - 1)); + var elem = document.getElementById('t' + i); + if (parent_elem && elem) { + elem.style.top = parent_elem.offsetHeight + parent_elem.offsetTop + 'px'; + } + } + } + } catch (error) { + console.error("Error processing message:", error); } sendResponse({}); diff --git a/Audio-Transcription-Chrome/manifest.json b/Audio-Transcription-Chrome/manifest.json index a46e026..2ec236f 100644 --- a/Audio-Transcription-Chrome/manifest.json +++ b/Audio-Transcription-Chrome/manifest.json @@ -1,14 +1,20 @@ - { +{ "manifest_version": 3, "name": "Audio Transcription", "version": "1.0.0", "description": "This extension captures the audio on the current tab, sends it to a server for transcription and shows the transcription in Real-time.", - + "options_page": "options.html", "background": { "service_worker": "background.js" }, + "web_accessible_resources": [ + { + "resources": ["audiopreprocessor.js"], + "matches": [""] + } + ], "permissions": [ "storage", "activeTab", diff --git a/Audio-Transcription-Chrome/options.js b/Audio-Transcription-Chrome/options.js index 6c3ef62..beee5bc 100644 --- a/Audio-Transcription-Chrome/options.js +++ b/Audio-Transcription-Chrome/options.js @@ -31,41 +31,6 @@ function sendMessageToTab(tabId, data) { }); } - -/** - * Resamples the audio data to a target sample rate of 16kHz. - * @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data. - * @param {number} [origSampleRate=44100] - The original sample rate of the audio data. - * @returns {Float32Array} The resampled audio data at 16kHz. - */ -function resampleTo16kHZ(audioData, origSampleRate = 44100) { - // Convert the audio data to a Float32Array - const data = new Float32Array(audioData); - - // Calculate the desired length of the resampled data - const targetLength = Math.round(data.length * (16000 / origSampleRate)); - - // Create a new Float32Array for the resampled data - const resampledData = new Float32Array(targetLength); - - // Calculate the spring factor and initialize the first and last values - const springFactor = (data.length - 1) / (targetLength - 1); - resampledData[0] = data[0]; - resampledData[targetLength - 1] = data[data.length - 1]; - - // Resample the audio data - for (let i = 1; i < targetLength - 1; i++) { - const index = i * springFactor; - const leftIndex = Math.floor(index).toFixed(); - const rightIndex = Math.ceil(index).toFixed(); - const fraction = index - leftIndex; - resampledData[i] = data[leftIndex] + (data[rightIndex] - data[leftIndex]) * fraction; - } - - // Return the resampled data - return resampledData; -} - function generateUUID() { let dt = new Date().getTime(); const uuid = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) { @@ -76,24 +41,99 @@ function generateUUID() { return uuid; } +// Global variables for audio processing +let audioContext = null; +let preNode = null; +let socket = null; +let isServerReady = false; +let currentStream = null; +let currentOptions = null; + +// AudioWorklet URL - make sure this path matches your manifest.json +const WORKLET_URL = chrome.runtime.getURL('audiopreprocessor.js'); + +async function initAudioWorklet(stream) { + audioContext = new AudioContext(); + if (audioContext.state === 'suspended') { + await audioContext.resume(); + } + + try { + await audioContext.audioWorklet.addModule(WORKLET_URL); + preNode = new AudioWorkletNode(audioContext, 'audiopreprocessor'); + const mediaStream = audioContext.createMediaStreamSource(stream); + + mediaStream.connect(preNode); + preNode.connect(audioContext.destination); + preNode.port.onmessage = (event) => { + const data = event.data; + + + const audio16k = data; // Float32Array @ 16 kHz + + if (socket && socket.readyState === WebSocket.OPEN && isServerReady) { + socket.send(audio16k); + } + }; + + // Test if we can hear audio (this will help verify the audio path) + + } catch (error) { + console.error("Error initializing AudioWorklet:", error); + throw error; + } +} + +function cleanupAudio() { + + if (preNode) { + preNode.port.onmessage = null; + preNode.disconnect(); + preNode = null; + } + + if (audioContext) { + audioContext.close(); + audioContext = null; + } + + if (currentStream) { + currentStream.getTracks().forEach(track => { + track.stop(); + console.log("Stopped track:", track.kind); + }); + currentStream = null; + } +} /** * Starts recording audio from the captured tab. * @param {Object} option - The options object containing the currentTabId. */ async function startRecord(option) { + currentOptions = option; const stream = await captureTabAudio(); const uuid = generateUUID(); if (stream) { - // call when the stream inactive + currentStream = stream; stream.oninactive = () => { + cleanupAudio(); window.close(); }; - const socket = new WebSocket(`ws://${option.host}:${option.port}/`); - let isServerReady = false; + + try { + await initAudioWorklet(stream); + } catch (error) { + console.error("Failed to initialize AudioWorklet:", error); + return; + } + + socket = new WebSocket(`ws://${option.host}:${option.port}/`); + isServerReady = false; let language = option.language; - socket.onopen = function(e) { + + socket.onopen = function(e) { socket.send( JSON.stringify({ uid: uuid, @@ -129,7 +169,6 @@ async function startRecord(option) { language = data["language"]; // send message to popup.js to update dropdown - // console.log(language); chrome.runtime.sendMessage({ action: "updateSelectedLanguage", detectedLanguage: language, @@ -139,43 +178,33 @@ async function startRecord(option) { } if (data["message"] === "DISCONNECT"){ - chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false }) + chrome.runtime.sendMessage({ action: "toggleCaptureButtons", data: false, saveCaptions: option.saveCaptions }); return; } - res = await sendMessageToTab(option.currentTabId, { + const res = await sendMessageToTab(option.currentTabId, { type: "transcript", - data: event.data, + data: { + data: event.data, + saveCaptions: option.saveCaptions, + }, }); }; - - const audioDataCache = []; - const context = new AudioContext(); - const mediaStream = context.createMediaStreamSource(stream); - const recorder = context.createScriptProcessor(4096, 1, 1); - - recorder.onaudioprocess = async (event) => { - if (!context || !isServerReady) return; - - const inputData = event.inputBuffer.getChannelData(0); - const audioData16kHz = resampleTo16kHZ(inputData, context.sampleRate); - - audioDataCache.push(inputData); - - socket.send(audioData16kHz); + socket.onclose = () => { + cleanupAudio(); + }; + + socket.onerror = (error) => { + cleanupAudio(); }; - // Prevent page mute - mediaStream.connect(recorder); - recorder.connect(context.destination); - mediaStream.connect(context.destination); - // } } else { window.close(); } } + /** * Listener for incoming messages from the extension's background script. * @param {Object} request - The message request object. diff --git a/Audio-Transcription-Chrome/popup.html b/Audio-Transcription-Chrome/popup.html index 1320336..7c8ccdc 100644 --- a/Audio-Transcription-Chrome/popup.html +++ b/Audio-Transcription-Chrome/popup.html @@ -19,6 +19,10 @@ +
+ + +
+
+ + +
diff --git a/Audio-Transcription-Firefox/popup.js b/Audio-Transcription-Firefox/popup.js index 61c37a5..b11adc3 100644 --- a/Audio-Transcription-Firefox/popup.js +++ b/Audio-Transcription-Firefox/popup.js @@ -4,6 +4,7 @@ document.addEventListener("DOMContentLoaded", function() { const useServerCheckbox = document.getElementById("useServerCheckbox"); const useVadCheckbox = document.getElementById("useVadCheckbox"); + const saveCaptionCheckbox = document.getElementById("saveCaptionCheckbox"); const languageDropdown = document.getElementById('languageDropdown'); const taskDropdown = document.getElementById('taskDropdown'); const modelSizeDropdown = document.getElementById('modelSizeDropdown'); @@ -41,6 +42,12 @@ document.addEventListener("DOMContentLoaded", function() { } }); + browser.storage.local.get("saveCaptionState", ({ saveCaptionState }) => { + if (saveCaptionState !== undefined) { + saveCaptionCheckbox.checked = saveCaptionState; + } + }); + browser.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => { if (storedLanguage !== undefined) { languageDropdown.value = storedLanguage; @@ -85,6 +92,7 @@ document.addEventListener("DOMContentLoaded", function() { task: selectedTask, modelSize: selectedModelSize, useVad: useVadCheckbox.checked, + saveCaption: saveCaptionCheckbox.checked, } }); toggleCaptureButtons(true); @@ -101,7 +109,7 @@ document.addEventListener("DOMContentLoaded", function() { stopButton.addEventListener("click", function() { browser.tabs.query({ active: true, currentWindow: true }) .then(function(tabs) { - browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture" }) + browser.tabs.sendMessage(tabs[0].id, { action: "stopCapture", data: {saveCaption: saveCaptionCheckbox.checked, } }) .then(function(response) { toggleCaptureButtons(false); browser.storage.local.set({ capturingState: { isCapturing: false } }) @@ -124,6 +132,7 @@ document.addEventListener("DOMContentLoaded", function() { stopButton.disabled = !isCapturing; useServerCheckbox.disabled = isCapturing; useVadCheckbox.disabled = isCapturing; + saveCaptionCheckbox.disabled = isCapturing; modelSizeDropdown.disabled = isCapturing; languageDropdown.disabled = isCapturing; taskDropdown.disabled = isCapturing; @@ -142,6 +151,11 @@ document.addEventListener("DOMContentLoaded", function() { browser.storage.local.set({ useVadState }); }); + saveCaptionCheckbox.addEventListener("change", () => { + const saveCaptionState = saveCaptionCheckbox.checked; + browser.storage.local.set({ saveCaptionState }); + }); + languageDropdown.addEventListener('change', function() { if (languageDropdown.value === "") { selectedLanguage = null;