Integrate OpenVINO backend
Signed-off-by: makaveli <vineet.suryan@collabora.com>
This commit is contained in:
makaveli
@@ -44,7 +44,7 @@ class ServeClientFasterWhisper(ServeClientBase):
|
||||
self.task = task
|
||||
self.initial_prompt = initial_prompt
|
||||
self.vad_parameters = vad_parameters or {"onset": 0.5}
|
||||
self.no_speech_thresh = 0.45
|
||||
|
||||
self.same_output_threshold = 10
|
||||
self.end_time_for_same_output = None
|
||||
|
||||
@@ -221,161 +221,3 @@ class ServeClientFasterWhisper(ServeClientBase):
|
||||
|
||||
if len(segments):
|
||||
self.send_transcription_to_client(segments)
|
||||
|
||||
def speech_to_text(self):
|
||||
"""
|
||||
Process an audio stream in an infinite loop, continuously transcribing the speech.
|
||||
|
||||
This method continuously receives audio frames, performs real-time transcription, and sends
|
||||
transcribed segments to the client via a WebSocket connection.
|
||||
|
||||
If the client's language is not detected, it waits for 30 seconds of audio input to make a language prediction.
|
||||
It utilizes the Whisper ASR model to transcribe the audio, continuously processing and streaming results. Segments
|
||||
are sent to the client in real-time, and a history of segments is maintained to provide context.Pauses in speech
|
||||
(no output from Whisper) are handled by showing the previous output for a set duration. A blank segment is added if
|
||||
there is no speech for a specified duration to indicate a pause.
|
||||
|
||||
Raises:
|
||||
Exception: If there is an issue with audio processing or WebSocket communication.
|
||||
|
||||
"""
|
||||
while True:
|
||||
if self.exit:
|
||||
logging.info("Exiting speech to text thread")
|
||||
break
|
||||
|
||||
if self.frames_np is None:
|
||||
continue
|
||||
|
||||
self.clip_audio_if_no_valid_segment()
|
||||
|
||||
input_bytes, duration = self.get_audio_chunk_for_processing()
|
||||
if duration < 1.0:
|
||||
time.sleep(0.1) # wait for audio chunks to arrive
|
||||
continue
|
||||
try:
|
||||
input_sample = input_bytes.copy()
|
||||
result = self.transcribe_audio(input_sample)
|
||||
|
||||
if result is None or self.language is None:
|
||||
self.timestamp_offset += duration
|
||||
time.sleep(0.25) # wait for voice activity, result is None when no voice activity
|
||||
continue
|
||||
self.handle_transcription_output(result, duration)
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"[ERROR]: Failed to transcribe audio chunk: {e}")
|
||||
time.sleep(0.01)
|
||||
|
||||
def format_segment(self, start, end, text, completed=False):
|
||||
"""
|
||||
Formats a transcription segment with precise start and end times alongside the transcribed text.
|
||||
|
||||
Args:
|
||||
start (float): The start time of the transcription segment in seconds.
|
||||
end (float): The end time of the transcription segment in seconds.
|
||||
text (str): The transcribed text corresponding to the segment.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary representing the formatted transcription segment, including
|
||||
'start' and 'end' times as strings with three decimal places and the 'text'
|
||||
of the transcription.
|
||||
"""
|
||||
return {
|
||||
'start': "{:.3f}".format(start),
|
||||
'end': "{:.3f}".format(end),
|
||||
'text': text,
|
||||
'completed': completed
|
||||
}
|
||||
|
||||
def update_segments(self, segments, duration):
|
||||
"""
|
||||
Processes the segments from whisper. Appends all the segments to the list
|
||||
except for the last segment assuming that it is incomplete.
|
||||
|
||||
Updates the ongoing transcript with transcribed segments, including their start and end times.
|
||||
Complete segments are appended to the transcript in chronological order. Incomplete segments
|
||||
(assumed to be the last one) are processed to identify repeated content. If the same incomplete
|
||||
segment is seen multiple times, it updates the offset and appends the segment to the transcript.
|
||||
A threshold is used to detect repeated content and ensure it is only included once in the transcript.
|
||||
The timestamp offset is updated based on the duration of processed segments. The method returns the
|
||||
last processed segment, allowing it to be sent to the client for real-time updates.
|
||||
|
||||
Args:
|
||||
segments(dict) : dictionary of segments as returned by whisper
|
||||
duration(float): duration of the current chunk
|
||||
|
||||
Returns:
|
||||
dict or None: The last processed segment with its start time, end time, and transcribed text.
|
||||
Returns None if there are no valid segments to process.
|
||||
"""
|
||||
offset = None
|
||||
self.current_out = ''
|
||||
last_segment = None
|
||||
|
||||
# process complete segments
|
||||
if len(segments) > 1 and segments[-1].no_speech_prob <= self.no_speech_thresh:
|
||||
for i, s in enumerate(segments[:-1]):
|
||||
text_ = s.text
|
||||
self.text.append(text_)
|
||||
with self.lock:
|
||||
start, end = self.timestamp_offset + s.start, self.timestamp_offset + min(duration, s.end)
|
||||
|
||||
if start >= end:
|
||||
continue
|
||||
if s.no_speech_prob > self.no_speech_thresh:
|
||||
continue
|
||||
|
||||
self.transcript.append(self.format_segment(start, end, text_, completed=True))
|
||||
offset = min(duration, s.end)
|
||||
|
||||
# only process the last segment if it satisfies the no_speech_thresh
|
||||
if segments[-1].no_speech_prob <= self.no_speech_thresh:
|
||||
self.current_out += segments[-1].text
|
||||
with self.lock:
|
||||
last_segment = self.format_segment(
|
||||
self.timestamp_offset + segments[-1].start,
|
||||
self.timestamp_offset + min(duration, segments[-1].end),
|
||||
self.current_out,
|
||||
completed=False
|
||||
)
|
||||
|
||||
if self.current_out.strip() == self.prev_out.strip() and self.current_out != '':
|
||||
self.same_output_count += 1
|
||||
|
||||
# if we remove the audio because of same output on the nth reptition we might remove the
|
||||
# audio thats not yet transcribed so, capturing the time when it was repeated for the first time
|
||||
if self.end_time_for_same_output is None:
|
||||
self.end_time_for_same_output = segments[-1].end
|
||||
time.sleep(0.1) # wait for some voice activity just in case there is an unitended pause from the speaker for better punctuations.
|
||||
else:
|
||||
self.same_output_count = 0
|
||||
self.end_time_for_same_output = None
|
||||
|
||||
# if same incomplete segment is seen multiple times then update the offset
|
||||
# and append the segment to the list
|
||||
if self.same_output_count > self.same_output_threshold:
|
||||
if not len(self.text) or self.text[-1].strip().lower() != self.current_out.strip().lower():
|
||||
self.text.append(self.current_out)
|
||||
with self.lock:
|
||||
self.transcript.append(self.format_segment(
|
||||
self.timestamp_offset,
|
||||
self.timestamp_offset + min(duration, self.end_time_for_same_output),
|
||||
self.current_out,
|
||||
completed=True
|
||||
))
|
||||
self.current_out = ''
|
||||
offset = min(duration, self.end_time_for_same_output)
|
||||
self.same_output_count = 0
|
||||
last_segment = None
|
||||
self.end_time_for_same_output = None
|
||||
else:
|
||||
self.prev_out = self.current_out
|
||||
|
||||
# update offset
|
||||
if offset is not None:
|
||||
with self.lock:
|
||||
self.timestamp_offset += offset
|
||||
|
||||
return last_segment
|
||||
|
||||
|
||||
Reference in New Issue
Block a user