Index
+ + +A
+| + |
B
+| + |
C
+| + | + |
D
+| + |
F
+| + |
G
+| + | + |
M
+
|
+
O
+| + | + |
P
+| + |
R
+| + | + |
S
+| + | + |
T
+| + | + |
U
+| + |
W
+| + | + |
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/html/search.html b/docs/html/search.html
new file mode 100644
index 0000000..0ae5276
--- /dev/null
+++ b/docs/html/search.html
@@ -0,0 +1,117 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Python Module Index
+ +
+ w
+
+
+ | + w | ||
| + |
+ whisper_live | + |
| + |
+ whisper_live.client | + |
| + |
+ whisper_live.server | + |
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/html/searchindex.js b/docs/html/searchindex.js
new file mode 100644
index 0000000..d75d57a
--- /dev/null
+++ b/docs/html/searchindex.js
@@ -0,0 +1 @@
+Search.setIndex({"docnames": ["index"], "filenames": ["index.rst"], "titles": ["Welcome to Whisper Live documentation!"], "terms": {"class": 0, "whisper_l": 0, "server": 0, "servecli": 0, "websocket": 0, "task": 0, "transcrib": 0, "devic": 0, "none": 0, "multilingu": 0, "fals": 0, "languag": 0, "client_uid": 0, "attribut": 0, "rate": 0, "int": 0, "The": 0, "audio": 0, "sampl": 0, "constant": 0, "set": 0, "16000": 0, "server_readi": 0, "str": 0, "A": 0, "messag": 0, "i": 0, "readi": 0, "disconnect": 0, "client": 0, "should": 0, "uniqu": 0, "identifi": 0, "data": 0, "byte": 0, "accumul": 0, "frame": 0, "transcript": 0, "type": 0, "e": 0, "g": 0, "whispermodel": 0, "model": 0, "speech": 0, "text": 0, "timestamp_offset": 0, "float": 0, "offset": 0, "timestamp": 0, "frames_np": 0, "numpi": 0, "ndarrai": 0, "arrai": 0, "store": 0, "frames_offset": 0, "list": 0, "segment": 0, "current_out": 0, "current": 0, "incomplet": 0, "prev_out": 0, "previou": 0, "t_start": 0, "start": 0, "exit": 0, "bool": 0, "flag": 0, "thread": 0, "same_output_threshold": 0, "threshold": 0, "consecut": 0, "same": 0, "output": 0, "show_prev_out_thresh": 0, "show": 0, "add_pause_thresh": 0, "ad": 0, "paus": 0, "blank": 0, "send_last_n_seg": 0, "number": 0, "last": 0, "send": 0, "wrapper": 0, "textwrap": 0, "textwrapp": 0, "format": 0, "pick_previous_seg": 0, "includ": 0, "connect": 0, "add_fram": 0, "frame_np": 0, "add": 0, "ongo": 0, "stream": 0, "buffer": 0, "thi": 0, "method": 0, "respons": 0, "maintain": 0, "allow": 0, "continu": 0, "addit": 0, "thei": 0, "ar": 0, "receiv": 0, "It": 0, "also": 0, "ensur": 0, "doe": 0, "exce": 0, "specifi": 0, "size": 0, "prevent": 0, "excess": 0, "memori": 0, "usag": 0, "If": 0, "45": 0, "second": 0, "discard": 0, "oldest": 0, "30": 0, "reason": 0, "empti": 0, "initi": 0, "provid": 0, "us": 0, "real": 0, "time": 0, "process": 0, "arg": 0, "cleanup": 0, "perform": 0, "befor": 0, "servic": 0, "necessari": 0, "stop": 0, "mark": 0, "gracefulli": 0, "destroi": 0, "resourc": 0, "associ": 0, "notifi": 0, "via": 0, "them": 0, "fill_output": 0, "combin": 0, "complet": 0, "result": 0, "wrap": 0, "two": 0, "line": 0, "each": 0, "contain": 0, "maximum": 0, "50": 0, "charact": 0, "fit": 0, "within": 0, "per": 0, "concaten": 0, "order": 0, "exist": 0, "most": 0, "recent": 0, "first": 0, "older": 0, "prepend": 0, "need": 0, "limit": 0, "3": 0, "detect": 0, "ani": 0, "preced": 0, "content": 0, "return": 0, "singl": 0, "string": 0, "speech_to_text": 0, "an": 0, "infinit": 0, "loop": 0, "": 0, "wait": 0, "input": 0, "make": 0, "predict": 0, "util": 0, "asr": 0, "sent": 0, "histori": 0, "context": 0, "from": 0, "handl": 0, "durat": 0, "rais": 0, "except": 0, "issu": 0, "commun": 0, "update_seg": 0, "append": 0, "all": 0, "assum": 0, "updat": 0, "end": 0, "chronolog": 0, "one": 0, "repeat": 0, "seen": 0, "multipl": 0, "onli": 0, "onc": 0, "base": 0, "dict": 0, "dictionari": 0, "chunk": 0, "its": 0, "valid": 0, "transcriptionserv": 0, "repres": 0, "incom": 0, "vad_model": 0, "torch": 0, "modul": 0, "voic": 0, "activ": 0, "vad_threshold": 0, "clients_start_tim": 0, "track": 0, "max_client": 0, "max_connection_tim": 0, "get_wait_tim": 0, "calcul": 0, "estim": 0, "minut": 0, "recv_audio": 0, "over": 0, "vad": 0, "determin": 0, "reach": 0, "statu": 0, "until": 0, "slot": 0, "avail": 0, "clean": 0, "up": 0, "error": 0, "dure": 0, "run": 0, "host": 0, "port": 0, "9090": 0, "address": 0, "bind": 0, "transcriptioncli": 0, "is_multilingu": 0, "lang": 0, "translat": 0, "act": 0, "high": 0, "level": 0, "can": 0, "hostnam": 0, "ip": 0, "option": 0, "whether": 0, "support": 0, "default": 0, "primari": 0, "which": 0, "english": 0, "en": 0, "requir": 0, "instanc": 0, "underli": 0, "exampl": 0, "To": 0, "creat": 0, "microphon": 0, "python": 0, "transcription_cli": 0, "localhost": 0, "true": 0, "resampl": 0, "file": 0, "sr": 0, "http": 0, "github": 0, "com": 0, "openai": 0, "blob": 0, "7858aa9c08d98f75575035ecd6481f462d66ca27": 0, "py": 0, "l22": 0, "open": 0, "read": 0, "mono": 0, "waveform": 0, "save": 0, "resampled_fil": 0, "index": 0, "search": 0, "page": 0, "record": 0, "static": 0, "bytes_to_float_arrai": 0, "audio_byt": 0, "convert": 0, "16": 0, "bit": 0, "pcm": 0, "normal": 0, "have": 0, "valu": 0, "between": 0, "1": 0, "np": 0, "close_websocket": 0, "close": 0, "join": 0, "attempt": 0, "self": 0, "client_socket": 0, "after": 0, "proper": 0, "termin": 0, "get_client_socket": 0, "get": 0, "socket": 0, "websocketapp": 0, "on_messag": 0, "w": 0, "callback": 0, "function": 0, "call": 0, "when": 0, "variou": 0, "on_open": 0, "successfulli": 0, "configur": 0, "uid": 0, "mode": 0, "select": 0, "play_fil": 0, "filenam": 0, "plai": 0, "through": 0, "simultan": 0, "pyaudio": 0, "playback": 0, "point": 0, "typic": 0, "you": 0, "want": 0, "pre": 0, "path": 0, "out_fil": 0, "output_record": 0, "wav": 0, "record_second": 0, "directori": 0, "separ": 0, "interrupt": 0, "keyboardinterrupt": 0, "press": 0, "ctrl": 0, "c": 0, "name": 0, "entir": 0, "send_packet_to_serv": 0, "packet": 0, "write_audio_frames_to_fil": 0, "file_nam": 0, "write": 0, "overwritten": 0, "correct": 0, "match": 0, "channel": 0, "width": 0, "written": 0, "write_output_record": 0, "n_audio_fil": 0, "individu": 0, "expect": 0, "locat": 0, "final": 0, "delet": 0}, "objects": {"whisper_live": [[0, 0, 0, "-", "client"], [0, 0, 0, "-", "server"]], "whisper_live.client": [[0, 1, 1, "", "Client"], [0, 1, 1, "", "TranscriptionClient"], [0, 3, 1, "", "resample"]], "whisper_live.client.Client": [[0, 2, 1, "", "bytes_to_float_array"], [0, 2, 1, "", "close_websocket"], [0, 2, 1, "", "get_client_socket"], [0, 2, 1, "", "on_message"], [0, 2, 1, "", "on_open"], [0, 2, 1, "", "play_file"], [0, 2, 1, "", "record"], [0, 2, 1, "", "send_packet_to_server"], [0, 2, 1, "", "write_audio_frames_to_file"], [0, 2, 1, "", "write_output_recording"]], "whisper_live.server": [[0, 1, 1, "", "ServeClient"], [0, 1, 1, "", "TranscriptionServer"]], "whisper_live.server.ServeClient": [[0, 2, 1, "", "add_frames"], [0, 2, 1, "", "cleanup"], [0, 2, 1, "", "disconnect"], [0, 2, 1, "", "fill_output"], [0, 2, 1, "", "speech_to_text"], [0, 2, 1, "", "update_segments"]], "whisper_live.server.TranscriptionServer": [[0, 2, 1, "", "get_wait_time"], [0, 2, 1, "", "recv_audio"], [0, 2, 1, "", "run"]]}, "objtypes": {"0": "py:module", "1": "py:class", "2": "py:method", "3": "py:function"}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "class", "Python class"], "2": ["py", "method", "Python method"], "3": ["py", "function", "Python function"]}, "titleterms": {"welcom": 0, "whisper": 0, "live": 0, "document": 0, "indic": 0, "tabl": 0}, "envversion": {"sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 60}, "alltitles": {"Welcome to Whisper Live documentation!": [[0, "welcome-to-whisper-live-documentation"]], "Indices and tables": [[0, "indices-and-tables"]]}, "indexentries": {"client (class in whisper_live.client)": [[0, "whisper_live.client.Client"]], "serveclient (class in whisper_live.server)": [[0, "whisper_live.server.ServeClient"]], "transcriptionclient (class in whisper_live.client)": [[0, "whisper_live.client.TranscriptionClient"]], "transcriptionserver (class in whisper_live.server)": [[0, "whisper_live.server.TranscriptionServer"]], "add_frames() (whisper_live.server.serveclient method)": [[0, "whisper_live.server.ServeClient.add_frames"]], "bytes_to_float_array() (whisper_live.client.client static method)": [[0, "whisper_live.client.Client.bytes_to_float_array"]], "cleanup() (whisper_live.server.serveclient method)": [[0, "whisper_live.server.ServeClient.cleanup"]], "close_websocket() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.close_websocket"]], "disconnect() (whisper_live.server.serveclient method)": [[0, "whisper_live.server.ServeClient.disconnect"]], "fill_output() (whisper_live.server.serveclient method)": [[0, "whisper_live.server.ServeClient.fill_output"]], "get_client_socket() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.get_client_socket"]], "get_wait_time() (whisper_live.server.transcriptionserver method)": [[0, "whisper_live.server.TranscriptionServer.get_wait_time"]], "module": [[0, "module-whisper_live.client"], [0, "module-whisper_live.server"]], "on_message() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.on_message"]], "on_open() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.on_open"]], "play_file() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.play_file"]], "record() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.record"]], "recv_audio() (whisper_live.server.transcriptionserver method)": [[0, "whisper_live.server.TranscriptionServer.recv_audio"]], "resample() (in module whisper_live.client)": [[0, "whisper_live.client.resample"]], "run() (whisper_live.server.transcriptionserver method)": [[0, "whisper_live.server.TranscriptionServer.run"]], "send_packet_to_server() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.send_packet_to_server"]], "speech_to_text() (whisper_live.server.serveclient method)": [[0, "whisper_live.server.ServeClient.speech_to_text"]], "update_segments() (whisper_live.server.serveclient method)": [[0, "whisper_live.server.ServeClient.update_segments"]], "whisper_live.client": [[0, "module-whisper_live.client"]], "whisper_live.server": [[0, "module-whisper_live.server"]], "write_audio_frames_to_file() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.write_audio_frames_to_file"]], "write_output_recording() (whisper_live.client.client method)": [[0, "whisper_live.client.Client.write_output_recording"]]}})
\ No newline at end of file
diff --git a/docs/index.html b/docs/index.html
new file mode 100644
index 0000000..5896b1d
--- /dev/null
+++ b/docs/index.html
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/whisper_live/client.py b/whisper_live/client.py
index 19e856a..ca2300c 100644
--- a/whisper_live/client.py
+++ b/whisper_live/client.py
@@ -18,12 +18,13 @@ def resample(file: str, sr: int = 16000):
# https://github.com/openai/whisper/blob/7858aa9c08d98f75575035ecd6481f462d66ca27/whisper/audio.py#L22
Open an audio file and read as mono waveform, resampling as necessary,
save the resampled audio
- Parameters
- ----------
- file: str
- The audio file to open
- sr: int
- The sample rate to resample the audio if necessary
+
+ Args:
+ file (str): The audio file to open
+ sr (int): The sample rate to resample the audio if necessary
+
+ Returns:
+ resampled_file (str): The resampled audio file
"""
try:
# This launches a subprocess to decode audio while down-mixing and resampling as necessary.
@@ -43,11 +44,28 @@ def resample(file: str, sr: int = 16000):
class Client:
+ """
+ Handles audio recording, streaming, and communication with a server using WebSocket.
+ """
INSTANCES = {}
def __init__(
self, host=None, port=None, is_multilingual=False, lang=None, translate=False
):
+ """
+ Initializes a Client instance for audio recording and streaming to a server.
+
+ If host and port are not provided, the WebSocket connection will not be established.
+ When translate is True, the task will be set to "translate" instead of "transcribe".
+ he audio recording starts immediately upon initialization.
+
+ Args:
+ host (str): The hostname or IP address of the server.
+ port (int): The port number for the WebSocket server.
+ is_multilingual (bool, optional): Specifies if multilingual transcription is enabled. Default is False.
+ lang (str, optional): The selected language for transcription when multilingual is disabled. Default is None.
+ translate (bool, optional): Specifies if the task is translation. Default is False.
+ """
self.chunk = 1024
self.format = pyaudio.paInt16
self.channels = 1
@@ -77,7 +95,6 @@ class Client:
frames_per_buffer=self.chunk,
)
- # create websocket connection
if host is not None and port is not None:
socket_url = f"ws://{host}:{port}"
self.client_socket = websocket.WebSocketApp(
@@ -104,6 +121,18 @@ class Client:
print("[INFO]: * recording")
def on_message(self, ws, message):
+ """
+ Callback function called when a message is received from the server.
+
+ It updates various attributes of the client based on the received message, including
+ recording status, language detection, and server messages. If a disconnect message
+ is received, it sets the recording status to False.
+
+ Args:
+ ws (websocket.WebSocketApp): The WebSocket client instance.
+ message (str): The received message from the server.
+
+ """
self.last_response_recieved = time.time()
message = json.loads(message)
@@ -164,6 +193,16 @@ class Client:
print(f"[INFO]: Websocket connection closed: {close_status_code}: {close_msg}")
def on_open(self, ws):
+ """
+ Callback function called when the WebSocket connection is successfully opened.
+
+ Sends an initial configuration message to the server, including client UID, multilingual mode,
+ language selection, and task type.
+
+ Args:
+ ws (websocket.WebSocketApp): The WebSocket client instance.
+
+ """
print(self.multilingual, self.language, self.task)
print("[INFO]: Opened connection")
@@ -180,16 +219,49 @@ class Client:
@staticmethod
def bytes_to_float_array(audio_bytes):
+ """
+ Convert audio data from bytes to a NumPy float array.
+
+ It assumes that the audio data is in 16-bit PCM format. The audio data is normalized to
+ have values between -1 and 1.
+
+ Args:
+ audio_bytes (bytes): Audio data in bytes.
+
+ Returns:
+ np.ndarray: A NumPy array containing the audio data as float values normalized between -1 and 1.
+ """
raw_data = np.frombuffer(buffer=audio_bytes, dtype=np.int16)
return raw_data.astype(np.float32) / 32768.0
def send_packet_to_server(self, message):
+ """
+ Send an audio packet to the server using WebSocket.
+
+ Args:
+ message (bytes): The audio data packet in bytes to be sent to the server.
+
+ """
try:
self.client_socket.send(message, websocket.ABNF.OPCODE_BINARY)
except Exception as e:
print(e)
def play_file(self, filename):
+ """
+ Play an audio file and send it to the server for processing.
+
+ Reads an audio file, plays it through the audio output, and simultaneously sends
+ the audio data to the server for processing. It uses PyAudio to create an audio
+ stream for playback. The audio data is read from the file in chunks, converted to
+ floating-point format, and sent to the server using WebSocket communication.
+ This method is typically used when you want to process pre-recorded audio and send it
+ to the server in real-time.
+
+ Args:
+ filename (str): The path to the audio file to be played and sent to the server.
+ """
+
# read audio and create pyaudio stream
with wave.open(filename, "rb") as wavfile:
self.stream = self.p.open(
@@ -213,8 +285,7 @@ class Client:
wavfile.close()
assert self.last_response_recieved
- elapsed_time = time.time() - self.last_response_recieved
- while elapsed_time < self.disconnect_if_no_response_for:
+ while time.time() - self.last_response_recieved < self.disconnect_if_no_response_for:
continue
self.stream.close()
self.close_websocket()
@@ -228,20 +299,44 @@ class Client:
print("[INFO]: Keyboard interrupt.")
def close_websocket(self):
+ """
+ Close the WebSocket connection and join the WebSocket thread.
+
+ First attempts to close the WebSocket connection using `self.client_socket.close()`. After
+ closing the connection, it joins the WebSocket thread to ensure proper termination.
+
+ """
try:
- self.client_socket.close() # Close the WebSocket connection
+ self.client_socket.close()
except Exception as e:
print("[ERROR]: Error closing WebSocket:", e)
try:
- self.ws_thread.join() # Wait for the WebSocket thread to finish
+ self.ws_thread.join()
except Exception as e:
print("[ERROR:] Error joining WebSocket thread:", e)
def get_client_socket(self):
+ """
+ Get the WebSocket client socket instance.
+
+ Returns:
+ WebSocketApp: The WebSocket client socket instance currently in use by the client.
+ """
return self.client_socket
def write_audio_frames_to_file(self, frames, file_name):
+ """
+ Write audio frames to a WAV file.
+
+ The WAV file is created or overwritten with the specified name. The audio frames should be
+ in the correct format and match the specified channel, sample width, and sample rate.
+
+ Args:
+ frames (bytes): The audio frames to be written to the file.
+ file_name (str): The name of the WAV file to which the frames will be written.
+
+ """
with wave.open(file_name, "wb") as wavfile:
wavfile: wave.Wave_write
wavfile.setnchannels(self.channels)
@@ -250,8 +345,23 @@ class Client:
wavfile.writeframes(frames)
def record(self, out_file="output_recording.wav"):
+ """
+ Record audio data from the input stream and save it to a WAV file.
+
+ Continuously records audio data from the input stream, sends it to the server via a WebSocket
+ connection, and simultaneously saves it to multiple WAV files in chunks. It stops recording when
+ the `RECORD_SECONDS` duration is reached or when the `RECORDING` flag is set to `False`.
+
+ Audio data is saved in chunks to the "chunks" directory. Each chunk is saved as a separate WAV file.
+ The recording will continue until the specified duration is reached or until the `RECORDING` flag is set to `False`.
+ The recording process can be interrupted by sending a KeyboardInterrupt (e.g., pressing Ctrl+C). After recording,
+ the method combines all the saved audio chunks into the specified `out_file`.
+
+ Args:
+ out_file (str, optional): The name of the output WAV file to save the entire recording. Default is "output_recording.wav".
+
+ """
n_audio_file = 0
- # create dir for saving audio chunks
if not os.path.exists("chunks"):
os.makedirs("chunks", exist_ok=True)
try:
@@ -289,10 +399,22 @@ class Client:
self.p.terminate()
self.close_websocket()
- # combine all the audio files
self.write_output_recording(n_audio_file, out_file)
def write_output_recording(self, n_audio_file, out_file):
+ """
+ Combine and save recorded audio chunks into a single WAV file.
+
+ The individual audio chunk files are expected to be located in the "chunks" directory. Reads each chunk
+ file, appends its audio data to the final recording, and then deletes the chunk file. After combining
+ and saving, the final recording is stored in the specified `out_file`.
+
+
+ Args:
+ n_audio_file (int): The number of audio chunk files to combine.
+ out_file (str): The name of the output WAV file to save the final recording.
+
+ """
input_files = [
f"chunks/{i}.wav"
for i in range(n_audio_file)
@@ -316,10 +438,44 @@ class Client:
class TranscriptionClient:
+ """
+ Client for handling audio transcription tasks via a WebSocket connection.
+
+ Acts as a high-level client for audio transcription tasks using a WebSocket connection. It can be used
+ to send audio data for transcription to a server and receive transcribed text segments.
+
+ Args:
+ host (str): The hostname or IP address of the server.
+ port (int): The port number to connect to on the server.
+ is_multilingual (bool, optional): Indicates whether the transcription should support multiple languages (default is False).
+ lang (str, optional): The primary language for transcription (used if `is_multilingual` is False). Default is None, which defaults to English ('en').
+ translate (bool, optional): Indicates whether translation tasks are required (default is False).
+
+ Attributes:
+ client (Client): An instance of the underlying Client class responsible for handling the WebSocket connection.
+
+ Example:
+ To create a TranscriptionClient and start transcription on microphone audio:
+ ```python
+ transcription_client = TranscriptionClient(host="localhost", port=9090, is_multilingual=True)
+ transcription_client()
+ ```
+ """
def __init__(self, host, port, is_multilingual=False, lang=None, translate=False):
self.client = Client(host, port, is_multilingual, lang, translate)
def __call__(self, audio=None):
+ """
+ Start the transcription process.
+
+ Initiates the transcription process by connecting to the server via a WebSocket. It waits for the server
+ to be ready to receive audio data and then sends audio for transcription. If an audio file is provided, it
+ will be played and streamed to the server; otherwise, it will perform live recording.
+
+ Args:
+ audio (str, optional): Path to an audio file for transcription. Default is None, which triggers live recording.
+
+ """
print("[INFO]: Waiting for server ready ...")
while not self.client.recording:
if self.client.waiting:
diff --git a/whisper_live/server.py b/whisper_live/server.py
index aa4439a..50b6381 100644
--- a/whisper_live/server.py
+++ b/whisper_live/server.py
@@ -1,16 +1,12 @@
import websockets
-import pickle, struct, time, pyaudio
+import time
import threading
-import os, json
-import base64
-import wave
+import json
import textwrap
import logging
# logging.basicConfig(level = logging.INFO)
-from collections import deque
-from dataclasses import dataclass
from websockets.sync.server import serve
import torch
@@ -24,39 +20,76 @@ class TranscriptionServer:
Represents a transcription server that handles incoming audio from clients.
Attributes:
+ RATE (int): The audio sampling rate (constant) set to 16000.
+ vad_model (torch.Module): The voice activity detection model.
+ vad_threshold (float): The voice activity detection threshold.
clients (dict): A dictionary to store connected clients.
+ websockets (dict): A dictionary to store WebSocket connections.
+ clients_start_time (dict): A dictionary to track client start times.
+ max_clients (int): Maximum allowed connected clients.
+ max_connection_time (int): Maximum allowed connection time in seconds.
"""
+
RATE = 16000
+
def __init__(self):
- # voice activity detection model
- self.vad_model, _ = torch.hub.load(repo_or_dir='snakers4/silero-vad',
- model='silero_vad',
- force_reload=True,
- onnx=True
- )
+ """
+ Initialize the TranscriptionServer.
+ """
+ # Load the voice activity detection model
+ self.vad_model, _ = torch.hub.load(
+ repo_or_dir='snakers4/silero-vad',
+ model='silero_vad',
+ force_reload=True,
+ onnx=True
+ )
+
+ # Voice activity detection threshold
self.vad_threshold = 0.4
+
self.clients = {}
self.websockets = {}
self.clients_start_time = {}
self.max_clients = 4
- self.max_connection_time = 600 # in seconds
-
+ self.max_connection_time = 600
+
def get_wait_time(self):
+ """
+ Calculate and return the estimated wait time for clients.
+
+ Returns:
+ float: The estimated wait time in minutes.
+ """
wait_time = None
- for k,v in self.clients_start_time.items():
+
+ for k, v in self.clients_start_time.items():
current_client_time_remaining = self.max_connection_time - (time.time() - v)
- if wait_time is None:
+
+ if wait_time is None or current_client_time_remaining < wait_time:
wait_time = current_client_time_remaining
- elif current_client_time_remaining < wait_time:
- wait_time = current_client_time_remaining
- return wait_time/60
+
+ return wait_time / 60
def recv_audio(self, websocket):
"""
Receive audio chunks from a client in an infinite loop.
+
+ Continuously receives audio frames from a connected client
+ over a WebSocket connection. It processes the audio frames using a
+ voice activity detection (VAD) model to determine if they contain speech
+ or not. If the audio frame contains speech, it is added to the client's
+ audio data for ASR.
+ If the maximum number of clients is reached, the method sends a
+ "WAIT" status to the client, indicating that they should wait
+ until a slot is available.
+ If a client's connection exceeds the maximum allowed time, it will
+ be disconnected, and the client's resources will be cleaned up.
Args:
websocket (WebSocket): The WebSocket connection for the client.
+
+ Raises:
+ Exception: If there is an error during the audio frame processing.
"""
logging.info("New client connected")
options = websocket.recv()
@@ -66,7 +99,7 @@ class TranscriptionServer:
logging.warning("Client Queue Full. Asking client to wait ...")
wait_time = self.get_wait_time()
response = {
- "uid" : options["uid"],
+ "uid": options["uid"],
"status": "WAIT",
"message": wait_time,
}
@@ -74,7 +107,7 @@ class TranscriptionServer:
websocket.close()
del websocket
return
-
+
client = ServeClient(
websocket,
multilingual=options["multilingual"],
@@ -82,9 +115,9 @@ class TranscriptionServer:
task=options["task"],
client_uid=options["uid"]
)
-
+
self.clients[websocket] = client
- self.clients_start_time[websocket] = time.time()
+ self.clients_start_time[websocket] = time.time()
while True:
try:
@@ -95,10 +128,11 @@ class TranscriptionServer:
speech_prob = self.vad_model(torch.from_numpy(frame_np.copy()), self.RATE).item()
if speech_prob < self.vad_threshold:
continue
-
+
except Exception as e:
logging.error(e)
return
+
self.clients[websocket].add_frames(frame_np)
elapsed_time = time.time() - self.clients_start_time[websocket]
@@ -112,7 +146,6 @@ class TranscriptionServer:
del websocket
break
-
except Exception as e:
logging.error(e)
self.clients[websocket].cleanup()
@@ -120,7 +153,6 @@ class TranscriptionServer:
self.clients_start_time.pop(websocket)
logging.info("Connection Closed.")
logging.info(self.clients)
-
del websocket
break
@@ -137,11 +169,54 @@ class TranscriptionServer:
class ServeClient:
+ """
+ Attributes:
+ RATE (int): The audio sampling rate (constant) set to 16000.
+ SERVER_READY (str): A constant message indicating that the server is ready.
+ DISCONNECT (str): A constant message indicating that the client should disconnect.
+ client_uid (str): A unique identifier for the client.
+ data (bytes): Accumulated audio data.
+ frames (bytes): Accumulated audio frames.
+ language (str): The language for transcription.
+ task (str): The task type, e.g., "transcribe."
+ transcriber (WhisperModel): The Whisper model for speech-to-text.
+ timestamp_offset (float): The offset in audio timestamps.
+ frames_np (numpy.ndarray): NumPy array to store audio frames.
+ frames_offset (float): The offset in audio frames.
+ text (list): List of transcribed text segments.
+ current_out (str): The current incomplete transcription.
+ prev_out (str): The previous incomplete transcription.
+ t_start (float): Timestamp for the start of transcription.
+ exit (bool): A flag to exit the transcription thread.
+ same_output_threshold (int): Threshold for consecutive same output segments.
+ show_prev_out_thresh (int): Threshold for showing previous output segments.
+ add_pause_thresh (int): Threshold for adding a pause (blank) segment.
+ transcript (list): List of transcribed segments.
+ send_last_n_segments (int): Number of last segments to send to the client.
+ wrapper (textwrap.TextWrapper): Text wrapper for formatting text.
+ pick_previous_segments (int): Number of previous segments to include in the output.
+ websocket: The WebSocket connection for the client.
+ """
RATE = 16000
SERVER_READY = "SERVER_READY"
DISCONNECT = "DISCONNECT"
def __init__(self, websocket, task="transcribe", device=None, multilingual=False, language=None, client_uid=None):
+ """
+ Initialize a ServeClient instance.
+ The Whisper model is initialized based on the client's language and device availability.
+ The transcription thread is started upon initialization. A "SERVER_READY" message is sent
+ to the client to indicate that the server is ready.
+
+ Args:
+ websocket (WebSocket): The WebSocket connection for the client.
+ task (str, optional): The task type, e.g., "transcribe." Defaults to "transcribe".
+ device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None.
+ multilingual (bool, optional): Whether the client supports multilingual transcription. Defaults to False.
+ language (str, optional): The language for transcription. Defaults to None.
+ client_uid (str, optional): A unique identifier for the client. Defaults to None.
+
+ """
self.client_uid = client_uid
self.data = b""
self.frames = b""
@@ -188,14 +263,21 @@ class ServeClient:
def fill_output(self, output):
"""
- Format output with current and previous complete segments
- into two lines of 50 characters.
+ Format the current incomplete transcription output by combining it with previous complete segments.
+ The resulting transcription is wrapped into two lines, each containing a maximum of 50 characters.
+
+ It ensures that the combined transcription fits within two lines, with a maximum of 50 characters per line.
+ Segments are concatenated in the order they exist in the list of previous segments, with the most
+ recent complete segment first and older segments prepended as needed to maintain the character limit.
+ If a 3-second pause is detected in the previous segments, any text preceding it is discarded to ensure
+ the transcription starts with the most recent complete content. The resulting transcription is returned
+ as a single string.
Args:
- output(str): current incomplete segment
+ output(str): The current incomplete transcription segment.
Returns:
- transcription wrapped in two lines
+ str: A formatted transcription wrapped in two lines.
"""
text = ''
pick_prev = min(len(self.text), self.pick_previous_segments)
@@ -209,6 +291,21 @@ class ServeClient:
return wrapped
def add_frames(self, frame_np):
+ """
+ Add audio frames to the ongoing audio stream buffer.
+
+ This method is responsible for maintaining the audio stream buffer, allowing the continuous addition
+ of audio frames as they are received. It also ensures that the buffer does not exceed a specified size
+ to prevent excessive memory usage.
+
+ If the buffer size exceeds a threshold (45 seconds of audio data), it discards the oldest 30 seconds
+ of audio data to maintain a reasonable buffer size. If the buffer is empty, it initializes it with the provided
+ audio frame. The audio stream buffer is used for real-time processing of audio data for transcription.
+
+ Args:
+ frame_np (numpy.ndarray): The audio frame data as a NumPy array.
+
+ """
if self.frames_np is not None and self.frames_np.shape[0] > 45*self.RATE:
self.frames_offset += 30.0
self.frames_np = self.frames_np[int(30*self.RATE):]
@@ -219,7 +316,20 @@ class ServeClient:
def speech_to_text(self):
"""
- Process audio stream in an infinite loop.
+ Process an audio stream in an infinite loop, continuously transcribing the speech.
+
+ This method continuously receives audio frames, performs real-time transcription, and sends
+ transcribed segments to the client via a WebSocket connection.
+
+ If the client's language is not detected, it waits for 30 seconds of audio input to make a language prediction.
+ It utilizes the Whisper ASR model to transcribe the audio, continuously processing and streaming results. Segments
+ are sent to the client in real-time, and a history of segments is maintained to provide context.Pauses in speech
+ (no output from Whisper) are handled by showing the previous output for a set duration. A blank segment is added if
+ there is no speech for a specified duration to indicate a pause.
+
+ Raises:
+ Exception: If there is an issue with audio processing or WebSocket communication.
+
"""
# detect language
if self.language is None:
@@ -327,12 +437,21 @@ class ServeClient:
Processes the segments from whisper. Appends all the segments to the list
except for the last segment assuming that it is incomplete.
+ Updates the ongoing transcript with transcribed segments, including their start and end times.
+ Complete segments are appended to the transcript in chronological order. Incomplete segments
+ (assumed to be the last one) are processed to identify repeated content. If the same incomplete
+ segment is seen multiple times, it updates the offset and appends the segment to the transcript.
+ A threshold is used to detect repeated content and ensure it is only included once in the transcript.
+ The timestamp offset is updated based on the duration of processed segments. The method returns the
+ last processed segment, allowing it to be sent to the client for real-time updates.
+
Args:
segments(dict) : dictionary of segments as returned by whisper
duration(float): duration of the current chunk
Returns:
- transcription for the current chunk
+ dict or None: The last processed segment with its start time, end time, and transcribed text.
+ Returns None if there are no valid segments to process.
"""
offset = None
self.current_out = ''
@@ -391,6 +510,13 @@ class ServeClient:
return last_segment
def disconnect(self):
+ """
+ Notify the client of disconnection and send a disconnect message.
+
+ This method sends a disconnect message to the client via the WebSocket connection to notify them
+ that the transcription service is disconnecting gracefully.
+
+ """
self.websocket.send(
json.dumps(
{
@@ -401,6 +527,14 @@ class ServeClient:
)
def cleanup(self):
+ """
+ Perform cleanup tasks before exiting the transcription service.
+
+ This method performs necessary cleanup tasks, including stopping the transcription thread, marking
+ the exit flag to indicate the transcription thread should exit gracefully, and destroying resources
+ associated with the transcription process.
+
+ """
logging.info("Cleaning up.")
self.exit = True
self.transcriber.destroy()
+
+
+
+
+
+
+
+
+
+
+
+ Search
+ + + + ++ Searching for multiple words only shows matches that contain + all words. +
+ + + + + + +
+
+
+
+
+