From 0854f158fc5dcffa21484b66f5ab20ef7f3ffafc Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 11 Jul 2023 00:32:55 +0800 Subject: [PATCH 1/3] load models before recieving audio --- server.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 4e18354..82f3d06 100644 --- a/server.py +++ b/server.py @@ -19,6 +19,7 @@ from transcriber import WhisperModel clients = {} +SERVER_READY = "SERVER_READY" def recv_audio(websocket): """ @@ -49,7 +50,12 @@ class ServeClient: self.payload_size = struct.calcsize("Q") self.data = b"" self.frames = b"" - self.transcriber = WhisperModel("small.en", compute_type="float16", local_files_only=False) + self.transcriber = WhisperModel( + "small.en", + device="cuda", + compute_type="float16", + local_files_only=False + ) # voice activity detection model self.vad_model, _ = torch.hub.load(repo_or_dir='snakers4/silero-vad', @@ -84,6 +90,7 @@ class ServeClient: self.websocket = websocket self.trans_thread = threading.Thread(target=self.speech_to_text) self.trans_thread.start() + self.websocket.send(json.dumps(SERVER_READY)) def fill_output(self, output): """ @@ -267,7 +274,6 @@ class ServeClient: logging.info("Cleaning up.") self.exit = True self.transcriber.destroy() - if __name__ == "__main__": From 6a2a070f583f4d8798dc9ea3c4b8a9e542cd91d9 Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Tue, 11 Jul 2023 00:36:35 +0800 Subject: [PATCH 2/3] update faster_whisper version req --- requirements/server.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/server.txt b/requirements/server.txt index 988f849..5e7084c 100644 --- a/requirements/server.txt +++ b/requirements/server.txt @@ -1,5 +1,5 @@ PyAudio -faster-whisper +faster-whisper==0.6.0 paho-mqtt --extra-index-url https://download.pytorch.org/whl/cu113 torch==1.12.1 From e32f597a42a3085830f2f9ff2254a28d4b15859e Mon Sep 17 00:00:00 2001 From: makaveli10 Date: Mon, 10 Jul 2023 22:14:04 +0530 Subject: [PATCH 3/3] wait for server to be ready with models --- Audio-Transcription-Firefox/content.js | 8 ++++++-- Audio-Transcription/options.js | 12 ++++++++++-- client.py | 20 ++++++++++++++++++-- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/Audio-Transcription-Firefox/content.js b/Audio-Transcription-Firefox/content.js index f8b06da..51ef0cd 100644 --- a/Audio-Transcription-Firefox/content.js +++ b/Audio-Transcription-Firefox/content.js @@ -45,8 +45,12 @@ function startRecording() { socket.send("handshake"); }; + let isServerReady = false; socket.onmessage = (event) => { - // console.log(event.data); + if (!isServerReady){ + isServerReady = true; + return; + } const data = event.data; browser.runtime.sendMessage({ action: "transcript", data }) .catch(function(error) { @@ -64,7 +68,7 @@ function startRecording() { recorder = audioContext.createScriptProcessor(4096, 1, 1); recorder.onaudioprocess = async (event) => { - if (!audioContext || !isCapturing) return; + if (!audioContext || !isCapturing || !isServerReady) return; const inputData = event.inputBuffer.getChannelData(0); const audioData16kHz = resampleTo16kHZ(inputData, audioContext.sampleRate); diff --git a/Audio-Transcription/options.js b/Audio-Transcription/options.js index 4134394..8f2092d 100644 --- a/Audio-Transcription/options.js +++ b/Audio-Transcription/options.js @@ -81,25 +81,32 @@ async function startRecord(option) { }; const socket = new WebSocket("ws://localhost:9090/"); + let isServerReady = false; socket.onopen = function(e) { socket.send("handshake"); }; socket.onmessage = async (event) => { - // console.log(event.data); + console.log(event.data); + if (isServerReady === false){ + isServerReady = true; + return; + } + res = await sendMessageToTab(option.currentTabId, { type: "transcript", data: event.data, }); }; + const audioDataCache = []; const context = new AudioContext(); const mediaStream = context.createMediaStreamSource(stream); const recorder = context.createScriptProcessor(4096, 1, 1); recorder.onaudioprocess = async (event) => { - if (!context) return; + if (!context || !isServerReady) return; const inputData = event.inputBuffer.getChannelData(0); const audioData16kHz = resampleTo16kHZ(inputData, context.sampleRate); @@ -114,6 +121,7 @@ async function startRecord(option) { mediaStream.connect(recorder); recorder.connect(context.destination); mediaStream.connect(context.destination); + // } } else { window.close(); } diff --git a/client.py b/client.py index 4dbdeab..62dfd83 100644 --- a/client.py +++ b/client.py @@ -17,11 +17,15 @@ FORMAT = pyaudio.paInt16 CHANNELS = 1 RATE = 16000 RECORD_SECONDS = 60000 - +START_RECORDING = False def on_message(ws, message): + global START_RECORDING message = json.loads(message) + if message == "SERVER_READY": + START_RECORDING = True + return text = [] if len(message): for seg in message: @@ -35,7 +39,10 @@ def on_message(ws, message): wrapper = textwrap.TextWrapper(width=60) word_list = wrapper.wrap(text="".join(text)) # Print each line. - os.system('clear') + if os.name=='nt': + os.system('cls') + else: + os.system('clear') for element in word_list: print(element) @@ -222,6 +229,15 @@ if __name__=="__main__": opt = parser.parse_args() c = Client(host=opt.host, port=opt.port) + # while loop to wait for server to be ready + print("Waiting for server ready ...") + while not START_RECORDING: + pass + if os.name=='nt': + os.system('cls') + else: + os.system('clear') + if opt.audio is not None: resampled_file = resample(opt.audio) c.play_file(resampled_file)