Aaron Boxer
686 lines
30 KiB
Python
686 lines
30 KiB
Python
import os
|
|
import time
|
|
import threading
|
|
import queue
|
|
import json
|
|
import functools
|
|
import logging
|
|
import shutil
|
|
import tempfile
|
|
from typing import Optional, List
|
|
from fastapi import FastAPI, UploadFile, Form
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from starlette.responses import PlainTextResponse, JSONResponse
|
|
import uvicorn
|
|
from faster_whisper import WhisperModel
|
|
import torch
|
|
|
|
from enum import Enum
|
|
from typing import List, Optional
|
|
import numpy as np
|
|
from websockets.sync.server import serve
|
|
from websockets.exceptions import ConnectionClosed
|
|
from whisper_live.vad import VoiceActivityDetector
|
|
from whisper_live.backend.base import ServeClientBase
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
|
|
class ClientManager:
|
|
def __init__(self, max_clients=4, max_connection_time=600):
|
|
"""
|
|
Initializes the ClientManager with specified limits on client connections and connection durations.
|
|
|
|
Args:
|
|
max_clients (int, optional): The maximum number of simultaneous client connections allowed. Defaults to 4.
|
|
max_connection_time (int, optional): The maximum duration (in seconds) a client can stay connected. Defaults
|
|
to 600 seconds (10 minutes).
|
|
"""
|
|
self.clients = {}
|
|
self.start_times = {}
|
|
self.max_clients = max_clients
|
|
self.max_connection_time = max_connection_time
|
|
self.lock = threading.Lock()
|
|
|
|
def add_client(self, websocket, client):
|
|
"""
|
|
Adds a client and their connection start time to the tracking dictionaries.
|
|
|
|
Args:
|
|
websocket: The websocket associated with the client to add.
|
|
client: The client object to be added and tracked.
|
|
"""
|
|
with self.lock:
|
|
self.clients[websocket] = client
|
|
self.start_times[websocket] = time.time()
|
|
|
|
def get_client(self, websocket):
|
|
"""
|
|
Retrieves a client associated with the given websocket.
|
|
|
|
Args:
|
|
websocket: The websocket associated with the client to retrieve.
|
|
|
|
Returns:
|
|
The client object if found, False otherwise.
|
|
"""
|
|
with self.lock:
|
|
if websocket in self.clients:
|
|
return self.clients[websocket]
|
|
return False
|
|
|
|
def remove_client(self, websocket):
|
|
"""
|
|
Removes a client and their connection start time from the tracking dictionaries. Performs cleanup on the
|
|
client if necessary.
|
|
|
|
Args:
|
|
websocket: The websocket associated with the client to be removed.
|
|
"""
|
|
with self.lock:
|
|
client = self.clients.pop(websocket, None)
|
|
self.start_times.pop(websocket, None)
|
|
if client:
|
|
client.cleanup()
|
|
|
|
def get_wait_time(self):
|
|
"""
|
|
Calculates the estimated wait time for new clients based on the remaining connection times of current clients.
|
|
|
|
Returns:
|
|
The estimated wait time in minutes for new clients to connect. Returns 0 if there are available slots.
|
|
"""
|
|
with self.lock:
|
|
wait_time = None
|
|
for start_time in self.start_times.values():
|
|
current_client_time_remaining = self.max_connection_time - (time.time() - start_time)
|
|
if wait_time is None or current_client_time_remaining < wait_time:
|
|
wait_time = current_client_time_remaining
|
|
return wait_time / 60 if wait_time is not None else 0
|
|
|
|
def is_server_full(self, websocket, options):
|
|
"""
|
|
Checks if the server is at its maximum client capacity and sends a wait message to the client if necessary.
|
|
|
|
Args:
|
|
websocket: The websocket of the client attempting to connect.
|
|
options: A dictionary of options that may include the client's unique identifier.
|
|
|
|
Returns:
|
|
True if the server is full, False otherwise.
|
|
"""
|
|
with self.lock:
|
|
if len(self.clients) >= self.max_clients:
|
|
wait_time = None
|
|
for start_time in self.start_times.values():
|
|
remaining = self.max_connection_time - (time.time() - start_time)
|
|
if wait_time is None or remaining < wait_time:
|
|
wait_time = remaining
|
|
wait_time_minutes = wait_time / 60 if wait_time is not None else 0
|
|
response = {"uid": options["uid"], "status": "WAIT", "message": wait_time_minutes}
|
|
websocket.send(json.dumps(response))
|
|
return True
|
|
return False
|
|
|
|
def is_client_timeout(self, websocket):
|
|
"""
|
|
Checks if a client has exceeded the maximum allowed connection time and disconnects them if so, issuing a warning.
|
|
|
|
Args:
|
|
websocket: The websocket associated with the client to check.
|
|
|
|
Returns:
|
|
True if the client's connection time has exceeded the maximum limit, False otherwise.
|
|
"""
|
|
with self.lock:
|
|
elapsed_time = time.time() - self.start_times[websocket]
|
|
client = self.clients.get(websocket)
|
|
if elapsed_time >= self.max_connection_time and client:
|
|
client.disconnect()
|
|
logging.warning(f"Client with uid '{client.client_uid}' disconnected due to overtime.")
|
|
return True
|
|
return False
|
|
|
|
|
|
class BackendType(Enum):
|
|
FASTER_WHISPER = "faster_whisper"
|
|
TENSORRT = "tensorrt"
|
|
OPENVINO = "openvino"
|
|
|
|
@staticmethod
|
|
def valid_types() -> List[str]:
|
|
return [backend_type.value for backend_type in BackendType]
|
|
|
|
@staticmethod
|
|
def is_valid(backend: str) -> bool:
|
|
return backend in BackendType.valid_types()
|
|
|
|
def is_faster_whisper(self) -> bool:
|
|
return self == BackendType.FASTER_WHISPER
|
|
|
|
def is_tensorrt(self) -> bool:
|
|
return self == BackendType.TENSORRT
|
|
|
|
def is_openvino(self) -> bool:
|
|
return self == BackendType.OPENVINO
|
|
|
|
|
|
class TranscriptionServer:
|
|
RATE = 16000
|
|
|
|
def __init__(self):
|
|
self.client_manager = None
|
|
self.no_voice_activity_chunks = 0
|
|
self.use_vad = True
|
|
self.single_model = False
|
|
self.batch_config = None
|
|
self.raw_pcm_input = False
|
|
|
|
def initialize_client(
|
|
self, websocket, options, faster_whisper_custom_model_path,
|
|
whisper_tensorrt_path, trt_multilingual, trt_py_session=False,
|
|
):
|
|
client: Optional[ServeClientBase] = None
|
|
|
|
# Check if client wants translation
|
|
enable_translation = options.get("enable_translation", False)
|
|
|
|
# Create translation queue if translation is enabled
|
|
translation_queue = None
|
|
translation_client = None
|
|
translation_thread = None
|
|
|
|
if enable_translation:
|
|
target_language = options.get("target_language", "fr")
|
|
translation_queue = queue.Queue(maxsize=ServeClientBase.MAX_TRANSLATION_QUEUE_SIZE)
|
|
from whisper_live.backend.translation_backend import ServeClientTranslation
|
|
translation_client = ServeClientTranslation(
|
|
client_uid=options["uid"],
|
|
websocket=websocket,
|
|
translation_queue=translation_queue,
|
|
target_language=target_language,
|
|
send_last_n_segments=options.get("send_last_n_segments", 10)
|
|
)
|
|
|
|
# Start translation thread
|
|
translation_thread = threading.Thread(
|
|
target=translation_client.speech_to_text,
|
|
daemon=True
|
|
)
|
|
translation_thread.start()
|
|
|
|
logging.info(f"Translation enabled for client {options['uid']} with target language: {target_language}")
|
|
|
|
if self.backend.is_tensorrt():
|
|
try:
|
|
from whisper_live.backend.trt_backend import ServeClientTensorRT
|
|
client = ServeClientTensorRT(
|
|
websocket,
|
|
multilingual=trt_multilingual,
|
|
language=options["language"],
|
|
task=options["task"],
|
|
client_uid=options["uid"],
|
|
model=whisper_tensorrt_path,
|
|
single_model=self.single_model,
|
|
use_py_session=trt_py_session,
|
|
send_last_n_segments=options.get("send_last_n_segments", 10),
|
|
no_speech_thresh=options.get("no_speech_thresh", 0.45),
|
|
clip_audio=options.get("clip_audio", False),
|
|
same_output_threshold=options.get("same_output_threshold", 10),
|
|
)
|
|
logging.info("Running TensorRT backend.")
|
|
except Exception as e:
|
|
logging.error(f"TensorRT-LLM not supported: {e}")
|
|
self.client_uid = options["uid"]
|
|
websocket.send(json.dumps({
|
|
"uid": self.client_uid,
|
|
"status": "WARNING",
|
|
"message": "TensorRT-LLM not supported on Server yet. "
|
|
"Reverting to available backend: 'faster_whisper'"
|
|
}))
|
|
self.backend = BackendType.FASTER_WHISPER
|
|
|
|
if self.backend.is_openvino():
|
|
try:
|
|
from whisper_live.backend.openvino_backend import ServeClientOpenVINO
|
|
client = ServeClientOpenVINO(
|
|
websocket,
|
|
language=options["language"],
|
|
task=options["task"],
|
|
client_uid=options["uid"],
|
|
model=options["model"],
|
|
single_model=self.single_model,
|
|
send_last_n_segments=options.get("send_last_n_segments", 10),
|
|
no_speech_thresh=options.get("no_speech_thresh", 0.45),
|
|
clip_audio=options.get("clip_audio", False),
|
|
same_output_threshold=options.get("same_output_threshold", 10),
|
|
)
|
|
logging.info("Running OpenVINO backend.")
|
|
except Exception as e:
|
|
logging.error(f"OpenVINO not supported: {e}")
|
|
self.backend = BackendType.FASTER_WHISPER
|
|
self.client_uid = options["uid"]
|
|
websocket.send(json.dumps({
|
|
"uid": self.client_uid,
|
|
"status": "WARNING",
|
|
"message": "OpenVINO not supported on Server yet. "
|
|
"Reverting to available backend: 'faster_whisper'"
|
|
}))
|
|
|
|
try:
|
|
if self.backend.is_faster_whisper():
|
|
from whisper_live.backend.faster_whisper_backend import ServeClientFasterWhisper
|
|
# model is of the form namespace/repo_name and not a filesystem path
|
|
if faster_whisper_custom_model_path is not None:
|
|
logging.info(f"Using custom model {faster_whisper_custom_model_path}")
|
|
options["model"] = faster_whisper_custom_model_path
|
|
client = ServeClientFasterWhisper(
|
|
websocket,
|
|
language=options["language"],
|
|
task=options["task"],
|
|
client_uid=options["uid"],
|
|
model=options["model"],
|
|
initial_prompt=options.get("initial_prompt"),
|
|
vad_parameters=options.get("vad_parameters"),
|
|
use_vad=self.use_vad,
|
|
single_model=self.single_model,
|
|
send_last_n_segments=options.get("send_last_n_segments", 10),
|
|
no_speech_thresh=options.get("no_speech_thresh", 0.45),
|
|
clip_audio=options.get("clip_audio", False),
|
|
same_output_threshold=options.get("same_output_threshold", 10),
|
|
cache_path=self.cache_path,
|
|
translation_queue=translation_queue,
|
|
hotwords=options.get("hotwords"),
|
|
)
|
|
|
|
logging.info("Running faster_whisper backend.")
|
|
|
|
# Start batch inference worker on first client (after model is loaded)
|
|
if (self.batch_config is not None
|
|
and ServeClientFasterWhisper.BATCH_WORKER is None
|
|
and ServeClientFasterWhisper.SINGLE_MODEL is not None):
|
|
from whisper_live.batch_inference import BatchInferenceWorker
|
|
worker = BatchInferenceWorker(
|
|
transcriber=ServeClientFasterWhisper.SINGLE_MODEL,
|
|
**self.batch_config,
|
|
)
|
|
worker.start()
|
|
ServeClientFasterWhisper.BATCH_WORKER = worker
|
|
except Exception as e:
|
|
logging.error(e)
|
|
return
|
|
|
|
if client is None:
|
|
raise ValueError(f"Backend type {self.backend.value} not recognised or not handled.")
|
|
|
|
if translation_client:
|
|
client.translation_client = translation_client
|
|
client.translation_thread = translation_thread
|
|
|
|
self.client_manager.add_client(websocket, client)
|
|
|
|
def get_audio_from_websocket(self, websocket):
|
|
"""
|
|
Receives audio buffer from websocket and creates a numpy array out of it.
|
|
|
|
Args:
|
|
websocket: The websocket to receive audio from.
|
|
|
|
Returns:
|
|
A numpy array containing the audio.
|
|
"""
|
|
frame_data = websocket.recv()
|
|
if frame_data == b"END_OF_AUDIO":
|
|
return False
|
|
if self.raw_pcm_input:
|
|
audio_np = np.frombuffer(frame_data, dtype=np.int16)
|
|
return audio_np.astype(np.float32) / 32768.0
|
|
return np.frombuffer(frame_data, dtype=np.float32)
|
|
|
|
def handle_new_connection(self, websocket, faster_whisper_custom_model_path,
|
|
whisper_tensorrt_path, trt_multilingual, trt_py_session=False):
|
|
try:
|
|
logging.info("New client connected")
|
|
options = websocket.recv()
|
|
options = json.loads(options)
|
|
|
|
self.use_vad = options.get('use_vad')
|
|
if self.client_manager.is_server_full(websocket, options):
|
|
websocket.close()
|
|
return False # Indicates that the connection should not continue
|
|
|
|
if self.backend.is_tensorrt():
|
|
self.vad_detector = VoiceActivityDetector(frame_rate=self.RATE)
|
|
self.initialize_client(websocket, options, faster_whisper_custom_model_path,
|
|
whisper_tensorrt_path, trt_multilingual, trt_py_session=trt_py_session)
|
|
return True
|
|
except json.JSONDecodeError:
|
|
logging.error("Failed to decode JSON from client")
|
|
return False
|
|
except ConnectionClosed:
|
|
logging.info("Connection closed by client")
|
|
return False
|
|
except Exception as e:
|
|
logging.error(f"Error during new connection initialization: {str(e)}")
|
|
return False
|
|
|
|
def process_audio_frames(self, websocket):
|
|
frame_np = self.get_audio_from_websocket(websocket)
|
|
client = self.client_manager.get_client(websocket)
|
|
if frame_np is False:
|
|
if self.backend.is_tensorrt():
|
|
client.set_eos(True)
|
|
return False
|
|
|
|
if self.backend.is_tensorrt():
|
|
voice_active = self.voice_activity(websocket, frame_np)
|
|
if voice_active:
|
|
self.no_voice_activity_chunks = 0
|
|
client.set_eos(False)
|
|
if self.use_vad and not voice_active:
|
|
return True
|
|
|
|
client.add_frames(frame_np)
|
|
return True
|
|
|
|
def recv_audio(self,
|
|
websocket,
|
|
backend: BackendType = BackendType.FASTER_WHISPER,
|
|
faster_whisper_custom_model_path=None,
|
|
whisper_tensorrt_path=None,
|
|
trt_multilingual=False,
|
|
trt_py_session=False):
|
|
"""
|
|
Receive audio chunks from a client in an infinite loop.
|
|
|
|
Continuously receives audio frames from a connected client
|
|
over a WebSocket connection. It processes the audio frames using a
|
|
voice activity detection (VAD) model to determine if they contain speech
|
|
or not. If the audio frame contains speech, it is added to the client's
|
|
audio data for ASR.
|
|
If the maximum number of clients is reached, the method sends a
|
|
"WAIT" status to the client, indicating that they should wait
|
|
until a slot is available.
|
|
If a client's connection exceeds the maximum allowed time, it will
|
|
be disconnected, and the client's resources will be cleaned up.
|
|
|
|
Args:
|
|
websocket (WebSocket): The WebSocket connection for the client.
|
|
backend (str): The backend to run the server with.
|
|
faster_whisper_custom_model_path (str): path to custom faster whisper model.
|
|
whisper_tensorrt_path (str): Required for tensorrt backend.
|
|
trt_multilingual(bool): Only used for tensorrt, True if multilingual model.
|
|
|
|
Raises:
|
|
Exception: If there is an error during the audio frame processing.
|
|
"""
|
|
self.backend = backend
|
|
if not self.handle_new_connection(websocket, faster_whisper_custom_model_path,
|
|
whisper_tensorrt_path, trt_multilingual, trt_py_session=trt_py_session):
|
|
return
|
|
|
|
try:
|
|
while not self.client_manager.is_client_timeout(websocket):
|
|
if not self.process_audio_frames(websocket):
|
|
break
|
|
except ConnectionClosed:
|
|
logging.info("Connection closed by client")
|
|
except Exception as e:
|
|
logging.error(f"Unexpected error: {str(e)}")
|
|
finally:
|
|
if self.client_manager.get_client(websocket):
|
|
self.cleanup(websocket)
|
|
websocket.close()
|
|
del websocket
|
|
|
|
def run(self,
|
|
host,
|
|
port=9090,
|
|
backend="tensorrt",
|
|
faster_whisper_custom_model_path=None,
|
|
whisper_tensorrt_path=None,
|
|
trt_multilingual=False,
|
|
trt_py_session=False,
|
|
single_model=False,
|
|
max_clients=4,
|
|
max_connection_time=600,
|
|
cache_path="~/.cache/whisper-live/",
|
|
rest_port=8000,
|
|
enable_rest=False,
|
|
cors_origins: Optional[str] = None,
|
|
batch_enabled=False,
|
|
batch_max_size=8,
|
|
batch_window_ms=50,
|
|
raw_pcm_input=False):
|
|
"""
|
|
Run the transcription server.
|
|
|
|
Args:
|
|
host (str): The host address to bind the server.
|
|
port (int): The port number to bind the server.
|
|
batch_enabled (bool): Enable cross-client GPU batch inference for
|
|
the faster_whisper backend. When enabled, ``single_model`` is
|
|
forced to True and a ``BatchInferenceWorker`` is started after
|
|
the first client connects. Defaults to False.
|
|
batch_max_size (int): Maximum number of requests per GPU batch.
|
|
Defaults to 8.
|
|
batch_window_ms (int): Maximum time in milliseconds to wait for
|
|
the batch to fill after the first request arrives. Defaults
|
|
to 50.
|
|
"""
|
|
self.cache_path = cache_path
|
|
self.raw_pcm_input = raw_pcm_input
|
|
|
|
if max_clients < 1:
|
|
raise ValueError(f"max_clients must be >= 1, got {max_clients}")
|
|
if max_connection_time <= 0:
|
|
raise ValueError(f"max_connection_time must be > 0, got {max_connection_time}")
|
|
if batch_enabled and batch_max_size < 1:
|
|
raise ValueError(f"batch_max_size must be >= 1, got {batch_max_size}")
|
|
if batch_enabled and batch_window_ms < 0:
|
|
raise ValueError(f"batch_window_ms must be >= 0, got {batch_window_ms}")
|
|
|
|
self.client_manager = ClientManager(max_clients, max_connection_time)
|
|
if faster_whisper_custom_model_path is not None and not os.path.exists(faster_whisper_custom_model_path):
|
|
if "/" not in faster_whisper_custom_model_path:
|
|
raise ValueError(f"Custom faster_whisper model '{faster_whisper_custom_model_path}' is not a valid path or HuggingFace model.")
|
|
if whisper_tensorrt_path is not None and not os.path.exists(whisper_tensorrt_path):
|
|
raise ValueError(f"TensorRT model '{whisper_tensorrt_path}' is not a valid path.")
|
|
|
|
# Batch inference config
|
|
if batch_enabled:
|
|
single_model = True # Batch mode requires shared model
|
|
self.batch_config = {
|
|
'max_batch_size': batch_max_size,
|
|
'batch_window_ms': batch_window_ms,
|
|
}
|
|
logging.info(f"Batch inference enabled (max_batch={batch_max_size}, window={batch_window_ms}ms)")
|
|
else:
|
|
self.batch_config = None
|
|
|
|
if single_model:
|
|
if faster_whisper_custom_model_path or whisper_tensorrt_path:
|
|
logging.info("Custom model option was provided. Switching to single model mode.")
|
|
self.single_model = True
|
|
# TODO: load model initially
|
|
else:
|
|
logging.info("Single model mode currently only works with custom models.")
|
|
if not BackendType.is_valid(backend):
|
|
raise ValueError(f"{backend} is not a valid backend type. Choose backend from {BackendType.valid_types()}")
|
|
|
|
# New OpenAI-compatible REST API (toggleable via enable_rest boolean)
|
|
if enable_rest:
|
|
app = FastAPI(title="WhisperLive OpenAI-Compatible API")
|
|
origins = [o.strip() for o in cors_origins.split(',')] if cors_origins else []
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=origins,
|
|
allow_credentials=True,
|
|
allow_methods=["*"], # Allows all methods (GET, POST, etc.)
|
|
allow_headers=["*"], # Allows all headers
|
|
)
|
|
|
|
|
|
@app.post("/v1/audio/transcriptions")
|
|
async def transcribe(
|
|
file: UploadFile,
|
|
model: str = Form(default="whisper-1"),
|
|
language: Optional[str] = Form(default=None),
|
|
prompt: Optional[str] = Form(default=None),
|
|
response_format: str = Form(default="json"),
|
|
temperature: float = Form(default=0.0),
|
|
timestamp_granularities: Optional[List[str]] = Form(default=None),
|
|
# Stubs for unsupported OpenAI params
|
|
chunking_strategy: Optional[str] = Form(default=None),
|
|
include: Optional[List[str]] = Form(default=None),
|
|
known_speaker_names: Optional[List[str]] = Form(default=None),
|
|
known_speaker_references: Optional[List[str]] = Form(default=None),
|
|
stream: bool = Form(default=False),
|
|
hotwords: Optional[str] = Form(default=None),
|
|
):
|
|
if stream:
|
|
return JSONResponse({"error": "Streaming not supported in this backend."}, status_code=400)
|
|
if chunking_strategy or known_speaker_names or known_speaker_references:
|
|
logging.warning("Diarization/chunking params ignored; not supported.")
|
|
|
|
supported_formats = ["json", "text", "srt", "verbose_json", "vtt"]
|
|
if response_format not in supported_formats:
|
|
return JSONResponse({"error": f"Unsupported response_format. Supported: {supported_formats}"}, status_code=400)
|
|
|
|
if model != "whisper-1":
|
|
logging.warning(f"Model '{model}' requested; using 'small' as fallback.")
|
|
model_name = faster_whisper_custom_model_path or "small"
|
|
|
|
try:
|
|
suffix = os.path.splitext(file.filename)[1] or ".wav"
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
|
|
shutil.copyfileobj(file.file, tmp)
|
|
tmp_path = tmp.name
|
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
compute_type = "float16" if device == "cuda" else "int8"
|
|
|
|
transcriber = WhisperModel(model_name, device=device, compute_type=compute_type)
|
|
segments, info = transcriber.transcribe(
|
|
tmp_path,
|
|
language=language,
|
|
initial_prompt=prompt,
|
|
temperature=temperature,
|
|
vad_filter=False,
|
|
word_timestamps=(timestamp_granularities and "word" in timestamp_granularities),
|
|
hotwords=hotwords,
|
|
)
|
|
|
|
text = " ".join([s.text.strip() for s in segments])
|
|
os.unlink(tmp_path)
|
|
|
|
if response_format == "text":
|
|
return PlainTextResponse(text)
|
|
elif response_format == "json":
|
|
return {"text": text}
|
|
elif response_format == "verbose_json":
|
|
verbose = {
|
|
"task": "transcribe",
|
|
"language": info.language,
|
|
"duration": info.duration,
|
|
"text": text,
|
|
"segments": []
|
|
}
|
|
for seg in segments:
|
|
seg_dict = {
|
|
"id": seg.id,
|
|
"seek": seg.seek,
|
|
"start": seg.start,
|
|
"end": seg.end,
|
|
"text": seg.text.strip(),
|
|
"tokens": seg.tokens,
|
|
"temperature": seg.temperature,
|
|
"avg_logprob": seg.avg_logprob,
|
|
"compression_ratio": seg.compression_ratio,
|
|
"no_speech_prob": seg.no_speech_prob
|
|
}
|
|
if timestamp_granularities and "word" in timestamp_granularities:
|
|
seg_dict["words"] = [{"word": w.word, "start": w.start, "end": w.end, "probability": w.probability} for w in seg.words]
|
|
verbose["segments"].append(seg_dict)
|
|
return verbose
|
|
elif response_format in ["srt", "vtt"]:
|
|
output = []
|
|
for i, seg in enumerate(segments, 1):
|
|
start = f"{int(seg.start // 3600):02}:{int((seg.start % 3600) // 60):02}:{seg.start % 60:06.3f}"
|
|
end = f"{int(seg.end // 3600):02}:{int((seg.end % 3600) // 60):02}:{seg.end % 60:06.3f}"
|
|
if response_format == "srt":
|
|
output.append(f"{i}\n{start.replace('.', ',')} --> {end.replace('.', ',')}\n{seg.text.strip()}\n")
|
|
else: # vtt
|
|
output.append(f"{start} --> {end}\n{seg.text.strip()}\n")
|
|
return PlainTextResponse("\n".join(output))
|
|
except Exception as e:
|
|
return JSONResponse({"error": str(e)}, status_code=500)
|
|
|
|
threading.Thread(
|
|
target=uvicorn.run,
|
|
args=(app,),
|
|
kwargs={"host": "0.0.0.0", "port": rest_port, "log_level": "info"},
|
|
daemon=True
|
|
).start()
|
|
logging.info(f"✅ OpenAI-Compatible API started on http://0.0.0.0:{rest_port}")
|
|
|
|
# Original WebSocket server (always supported)
|
|
with serve(
|
|
functools.partial(
|
|
self.recv_audio,
|
|
backend=BackendType(backend),
|
|
faster_whisper_custom_model_path=faster_whisper_custom_model_path,
|
|
whisper_tensorrt_path=whisper_tensorrt_path,
|
|
trt_multilingual=trt_multilingual,
|
|
trt_py_session=trt_py_session,
|
|
),
|
|
host,
|
|
port
|
|
) as server:
|
|
server.serve_forever()
|
|
|
|
def voice_activity(self, websocket, frame_np):
|
|
"""
|
|
Evaluates the voice activity in a given audio frame and manages the state of voice activity detection.
|
|
|
|
This method uses the configured voice activity detection (VAD) model to assess whether the given audio frame
|
|
contains speech. If the VAD model detects no voice activity for more than three consecutive frames,
|
|
it sets an end-of-speech (EOS) flag for the associated client. This method aims to efficiently manage
|
|
speech detection to improve subsequent processing steps.
|
|
|
|
Args:
|
|
websocket: The websocket associated with the current client. Used to retrieve the client object
|
|
from the client manager for state management.
|
|
frame_np (numpy.ndarray): The audio frame to be analyzed. This should be a NumPy array containing
|
|
the audio data for the current frame.
|
|
|
|
Returns:
|
|
bool: True if voice activity is detected in the current frame, False otherwise. When returning False
|
|
after detecting no voice activity for more than three consecutive frames, it also triggers the
|
|
end-of-speech (EOS) flag for the client.
|
|
"""
|
|
if not self.vad_detector(frame_np):
|
|
self.no_voice_activity_chunks += 1
|
|
if self.no_voice_activity_chunks > 3:
|
|
client = self.client_manager.get_client(websocket)
|
|
if not client.eos:
|
|
client.set_eos(True)
|
|
time.sleep(0.1) # Sleep 100m; wait some voice activity.
|
|
return False
|
|
return True
|
|
|
|
def cleanup(self, websocket):
|
|
"""
|
|
Cleans up resources associated with a given client's websocket.
|
|
|
|
Args:
|
|
websocket: The websocket associated with the client to be cleaned up.
|
|
"""
|
|
client = self.client_manager.get_client(websocket)
|
|
if client:
|
|
if hasattr(client, 'translation_client') and client.translation_client:
|
|
client.translation_client.cleanup()
|
|
|
|
# Wait for translation thread to finish
|
|
if hasattr(client, 'translation_thread') and client.translation_thread:
|
|
client.translation_thread.join(timeout=2.0)
|
|
self.client_manager.remove_client(websocket) |