add firefox extension
This commit is contained in:
makaveli10
@@ -0,0 +1,283 @@
|
||||
console.log("Content script injected.");
|
||||
let socket = null;
|
||||
let isCapturing = false;
|
||||
let mediaStream = null;
|
||||
let audioContext = null;
|
||||
let scriptProcessor = null;
|
||||
|
||||
/**
|
||||
* Resamples the audio data to a target sample rate of 16kHz.
|
||||
* @param {Array|ArrayBuffer|TypedArray} audioData - The input audio data.
|
||||
* @param {number} [origSampleRate=44100] - The original sample rate of the audio data.
|
||||
* @returns {Float32Array} The resampled audio data at 16kHz.
|
||||
*/
|
||||
function resampleTo16kHZ(audioData, origSampleRate = 44100) {
|
||||
// Convert the audio data to a Float32Array
|
||||
const data = new Float32Array(audioData);
|
||||
|
||||
// Calculate the desired length of the resampled data
|
||||
const targetLength = Math.round(data.length * (16000 / origSampleRate));
|
||||
|
||||
// Create a new Float32Array for the resampled data
|
||||
const resampledData = new Float32Array(targetLength);
|
||||
|
||||
// Calculate the spring factor and initialize the first and last values
|
||||
const springFactor = (data.length - 1) / (targetLength - 1);
|
||||
resampledData[0] = data[0];
|
||||
resampledData[targetLength - 1] = data[data.length - 1];
|
||||
|
||||
// Resample the audio data
|
||||
for (let i = 1; i < targetLength - 1; i++) {
|
||||
const index = i * springFactor;
|
||||
const leftIndex = Math.floor(index).toFixed();
|
||||
const rightIndex = Math.ceil(index).toFixed();
|
||||
const fraction = index - leftIndex;
|
||||
resampledData[i] = data[leftIndex] + (data[rightIndex] - data[leftIndex]) * fraction;
|
||||
}
|
||||
|
||||
// Return the resampled data
|
||||
return resampledData;
|
||||
}
|
||||
|
||||
function startRecording() {
|
||||
socket = new WebSocket("ws://localhost:9090/");
|
||||
socket.onopen = function(e) {
|
||||
socket.send("handshake");
|
||||
};
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
// console.log(event.data);
|
||||
const data = event.data;
|
||||
browser.runtime.sendMessage({ action: "transcript", data })
|
||||
.catch(function(error) {
|
||||
console.error("Error sending message:", error);
|
||||
});
|
||||
};
|
||||
|
||||
// Access the audio stream from the current tab
|
||||
navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
.then(function(stream) {
|
||||
// Create a new MediaRecorder instance
|
||||
const audioDataCache = [];
|
||||
audioContext = new AudioContext();
|
||||
mediaStream = audioContext.createMediaStreamSource(stream);
|
||||
recorder = audioContext.createScriptProcessor(4096, 1, 1);
|
||||
|
||||
recorder.onaudioprocess = async (event) => {
|
||||
if (!audioContext || !isCapturing) return;
|
||||
|
||||
const inputData = event.inputBuffer.getChannelData(0);
|
||||
const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate);
|
||||
|
||||
audioDataCache.push(inputData);
|
||||
|
||||
// feed inputs and run
|
||||
socket.send(audioData16kHz);
|
||||
};
|
||||
|
||||
// Prevent page mute
|
||||
mediaStream.connect(recorder);
|
||||
recorder.connect(audioContext.destination);
|
||||
})
|
||||
}
|
||||
|
||||
var elem_container = null;
|
||||
var elem_text = null;
|
||||
|
||||
var segments = [];
|
||||
var text_segments = [];
|
||||
|
||||
function init_element() {
|
||||
if (document.getElementById('transcription')) {
|
||||
return;
|
||||
}
|
||||
|
||||
elem_container = document.createElement('div');
|
||||
elem_container.id = "transcription";
|
||||
elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;top:0px;position:absolute;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;';
|
||||
|
||||
for (var i = 0; i < 4; i++) {
|
||||
elem_text = document.createElement('span');
|
||||
elem_text.style.cssText = 'position: absolute;padding-left:16px;padding-right:16px;';
|
||||
elem_text.id = "t" + i;
|
||||
elem_container.appendChild(elem_text);
|
||||
|
||||
if (i == 3) {
|
||||
elem_text.style.top = "-1000px"
|
||||
}
|
||||
}
|
||||
|
||||
document.body.appendChild(elem_container);
|
||||
|
||||
let x = 0;
|
||||
let y = 0;
|
||||
|
||||
// Query the element
|
||||
const ele = elem_container;
|
||||
|
||||
// Handle the mousedown event
|
||||
// that's triggered when user drags the element
|
||||
const mouseDownHandler = function (e) {
|
||||
// Get the current mouse position
|
||||
x = e.clientX;
|
||||
y = e.clientY;
|
||||
|
||||
// Attach the listeners to `document`
|
||||
document.addEventListener('mousemove', mouseMoveHandler);
|
||||
document.addEventListener('mouseup', mouseUpHandler);
|
||||
};
|
||||
|
||||
const mouseMoveHandler = function (e) {
|
||||
// How far the mouse has been moved
|
||||
const dx = e.clientX - x;
|
||||
const dy = e.clientY - y;
|
||||
|
||||
// Set the position of element
|
||||
ele.style.top = `${ele.offsetTop + dy}px`;
|
||||
ele.style.left = `${ele.offsetLeft + dx}px`;
|
||||
|
||||
// Reassign the position of mouse
|
||||
x = e.clientX;
|
||||
y = e.clientY;
|
||||
};
|
||||
|
||||
const mouseUpHandler = function () {
|
||||
// Remove the handlers of `mousemove` and `mouseup`
|
||||
document.removeEventListener('mousemove', mouseMoveHandler);
|
||||
document.removeEventListener('mouseup', mouseUpHandler);
|
||||
};
|
||||
|
||||
ele.addEventListener('mousedown', mouseDownHandler);
|
||||
}
|
||||
|
||||
function getStyle(el,styleProp)
|
||||
{
|
||||
var x = document.getElementById(el);
|
||||
if (x.currentStyle)
|
||||
var y = x.currentStyle[styleProp];
|
||||
else if (window.getComputedStyle)
|
||||
var y = document.defaultView.getComputedStyle(x,null).getPropertyValue(styleProp);
|
||||
return y;
|
||||
}
|
||||
|
||||
function get_lines(elem, line_height) {
|
||||
var divHeight = elem.offsetHeight;
|
||||
var lines = divHeight / line_height;
|
||||
|
||||
var original_text = elem.innerHTML;
|
||||
|
||||
var words = original_text.split(' ');
|
||||
var segments = [];
|
||||
var current_lines = 1;
|
||||
var segment = '';
|
||||
var segment_len = 0;
|
||||
for (var i = 0; i < words.length; i++)
|
||||
{
|
||||
segment += words[i] + ' ';
|
||||
elem.innerHTML = segment;
|
||||
divHeight = elem.offsetHeight;
|
||||
|
||||
if ((divHeight / line_height) > current_lines) {
|
||||
var line_segment = segment.substring(segment_len, segment.length - 1 - words[i].length - 1);
|
||||
segments.push(line_segment);
|
||||
segment_len += line_segment.length + 1;
|
||||
current_lines++;
|
||||
}
|
||||
}
|
||||
|
||||
var line_segment = segment.substring(segment_len, segment.length - 1)
|
||||
segments.push(line_segment);
|
||||
|
||||
elem.innerHTML = original_text;
|
||||
|
||||
return segments;
|
||||
|
||||
}
|
||||
|
||||
function remove_element() {
|
||||
var elem = document.getElementById('transcription')
|
||||
for (var i = 0; i < 4; i++) {
|
||||
document.getElementById("t" + i).remove();
|
||||
}
|
||||
elem.remove()
|
||||
}
|
||||
|
||||
|
||||
browser.runtime.onMessage.addListener((request, sender, sendResponse) => {
|
||||
const { action, data } = request;
|
||||
if (action === "startCapture") {
|
||||
isCapturing = true;
|
||||
startRecording();
|
||||
} else if (action === "stopCapture") {
|
||||
|
||||
isCapturing = false;
|
||||
if (socket) {
|
||||
socket.close();
|
||||
socket = null;
|
||||
}
|
||||
|
||||
if (audioContext) {
|
||||
audioContext.close();
|
||||
audioContext = null;
|
||||
mediaStream = null;
|
||||
recorder = null;
|
||||
}
|
||||
|
||||
|
||||
remove_element();
|
||||
|
||||
} else if (action === "show_transcript"){
|
||||
if (!isCapturing) return;
|
||||
init_element();
|
||||
message = JSON.parse(data);
|
||||
|
||||
var text = '';
|
||||
for (var i = 0; i < message.length; i++) {
|
||||
text += message[i].text + ' ';
|
||||
}
|
||||
text = text.replace(/(\r\n|\n|\r)/gm, "");
|
||||
|
||||
var elem = document.getElementById('t3');
|
||||
elem.innerHTML = text;
|
||||
|
||||
var line_height_style = getStyle('t3', 'line-height');
|
||||
var line_height = parseInt(line_height_style.substring(0, line_height_style.length - 2));
|
||||
var divHeight = elem.offsetHeight;
|
||||
var lines = divHeight / line_height;
|
||||
|
||||
text_segments = [];
|
||||
text_segments = get_lines(elem, line_height);
|
||||
|
||||
elem.innerHTML = '';
|
||||
|
||||
if (text_segments.length > 2) {
|
||||
for (var i = 0; i < 3; i++) {
|
||||
document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
|
||||
}
|
||||
} else {
|
||||
for (var i = 0; i < 3; i++) {
|
||||
document.getElementById('t' + i).innerHTML = '';
|
||||
}
|
||||
}
|
||||
|
||||
if (text_segments.length <= 2) {
|
||||
for (var i = 0; i < text_segments.length; i++) {
|
||||
document.getElementById('t' + i).innerHTML = text_segments[i];
|
||||
}
|
||||
} else {
|
||||
for (var i = 0; i < 3; i++) {
|
||||
document.getElementById('t' + i).innerHTML = text_segments[text_segments.length - 3 + i];
|
||||
}
|
||||
}
|
||||
|
||||
for (var i = 1; i < 3; i++)
|
||||
{
|
||||
var parent_elem = document.getElementById('t' + (i - 1));
|
||||
var elem = document.getElementById('t' + i);
|
||||
elem.style.top = parent_elem.offsetHeight + parent_elem.offsetTop + 'px';
|
||||
}
|
||||
}
|
||||
sendResponse({});
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user