cjones/WhisperLive
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #123 from lightwastak3n/main

Browse Source 
Chrome extension update; remove multililingual option faster-whisper
This commit is contained in:
makaveli
2024-02-03 19:24:45 +05:30
committed by GitHub
parent d89b27b8aa e697574870
commit 08fa183ba4
6 changed files with 120 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Audio-Transcription-Chrome/content.js
+1 -1
View File
@@ -59,7 +59,7 @@ function init_element() {
elem_container = document.createElement('div'); elem_container = document.createElement('div');
elem_container.id = "transcription"; elem_container.id = "transcription";
elem_container.style.cssText = 'padding-top:16px;font-size:18px;line-height:18px;top:0px;position:absolute;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;'; elem_container.style.cssText = 'padding-top:16px;font-size:18px;position: fixed; top: 50%; left: 50%; transform: translate(-50%, -50%);z-index: 9999;line-height:18px;width:500px;height:90px;opacity:0.9;z-index:100;background:black;border-radius:10px;color:white;';
for (var i = 0; i < 4; i++) { for (var i = 0; i < 4; i++) {
elem_text = document.createElement('span'); elem_text = document.createElement('span');
Audio-Transcription-Chrome/options.js
-4
View File
@@ -93,14 +93,10 @@ async function startRecord(option) {
const socket = new WebSocket(`ws://${option.host}:${option.port}/`); const socket = new WebSocket(`ws://${option.host}:${option.port}/`);
let isServerReady = false; let isServerReady = false;
let language = option.language; let language = option.language;
if (language === null && !option.multilingual) {
language = 'en';
}
socket.onopen = function(e) { socket.onopen = function(e) {
socket.send( socket.send(
JSON.stringify({ JSON.stringify({
uid: uuid, uid: uuid,
multilingual: option.multilingual,
language: option.language, language: option.language,
task: option.task, task: option.task,
model: option.modelSize model: option.modelSize
Audio-Transcription-Chrome/popup.html
+101 -100
View File
@@ -15,112 +15,109 @@
<input type="checkbox" id="useServerCheckbox"> <input type="checkbox" id="useServerCheckbox">
<label for="useServerCheckbox">Use Collabora Whisper-Live Server</label> <label for="useServerCheckbox">Use Collabora Whisper-Live Server</label>
</div> </div>
<div class="checkbox-container">
<input type="checkbox" id="useMultilingualCheckbox">
<label for="useMultilingualCheckbox">Use Multilingual Model</label>
</div>
<div class="dropdown-container"> <div class="dropdown-container">
<label for="languageDropdown">Select Language:</label> <label for="languageDropdown">Select Language:</label>
<select id="languageDropdown" disabled> <select id="languageDropdown">
<option value="">Select Language</option> <option value="" selected>Automatically detect</option>
<option value="zh">Chinese</option>
<option value="de">German</option>
<option value="es">Spanish</option>
<option value="ru">Russian</option>
<option value="ko">Korean</option>
<option value="fr">French</option>
<option value="ja">Japanese</option>
<option value="pt">Portuguese</option>
<option value="tr">Turkish</option>
<option value="pl">Polish</option>
<option value="ca">Catalan</option>
<option value="nl">Dutch</option>
<option value="ar">Arabic</option>
<option value="sv">Swedish</option>
<option value="it">Italian</option>
<option value="id">Indonesian</option>
<option value="hi">Hindi</option>
<option value="fi">Finnish</option>
<option value="vi">Vietnamese</option>
<option value="he">Hebrew</option>
<option value="uk">Ukrainian</option>
<option value="el">Greek</option>
<option value="ms">Malay</option>
<option value="cs">Czech</option>
<option value="ro">Romanian</option>
<option value="da">Danish</option>
<option value="hu">Hungarian</option>
<option value="ta">Tamil</option>
<option value="no">Norwegian</option>
<option value="th">Thai</option>
<option value="ur">Urdu</option>
<option value="hr">Croatian</option>
<option value="bg">Bulgarian</option>
<option value="lt">Lithuanian</option>
<option value="la">Latin</option>
<option value="mi">Maori</option>
<option value="ml">Malayalam</option>
<option value="cy">Welsh</option>
<option value="sk">Slovak</option>
<option value="te">Telugu</option>
<option value="fa">Persian</option>
<option value="lv">Latvian</option>
<option value="bn">Bengali</option>
<option value="sr">Serbian</option>
<option value="az">Azerbaijani</option>
<option value="sl">Slovenian</option>
<option value="kn">Kannada</option>
<option value="et">Estonian</option>
<option value="mk">Macedonian</option>
<option value="br">Breton</option>
<option value="eu">Basque</option>
<option value="is">Icelandic</option>
<option value="hy">Armenian</option>
<option value="ne">Nepali</option>
<option value="mn">Mongolian</option>
<option value="bs">Bosnian</option>
<option value="kk">Kazakh</option>
<option value="sq">Albanian</option>
<option value="sw">Swahili</option>
<option value="gl">Galician</option>
<option value="mr">Marathi</option>
<option value="pa">Punjabi</option>
<option value="si">Sinhala</option>
<option value="km">Khmer</option>
<option value="sn">Shona</option>
<option value="yo">Yoruba</option>
<option value="so">Somali</option>
<option value="af">Afrikaans</option> <option value="af">Afrikaans</option>
<option value="oc">Occitan</option> <option value="sq">Albanian</option>
<option value="ka">Georgian</option>
<option value="be">Belarusian</option>
<option value="tg">Tajik</option>
<option value="sd">Sindhi</option>
<option value="gu">Gujarati</option>
<option value="am">Amharic</option> <option value="am">Amharic</option>
<option value="yi">Yiddish</option> <option value="ar">Arabic</option>
<option value="lo">Lao</option> <option value="hy">Armenian</option>
<option value="uz">Uzbek</option>
<option value="fo">Faroese</option>
<option value="ht">Haitian Creole</option>
<option value="ps">Pashto</option>
<option value="tk">Turkmen</option>
<option value="nn">Nynorsk</option>
<option value="mt">Maltese</option>
<option value="sa">Sanskrit</option>
<option value="lb">Luxembourgish</option>
<option value="my">Myanmar</option>
<option value="bo">Tibetan</option>
<option value="tl">Tagalog</option>
<option value="mg">Malagasy</option>
<option value="as">Assamese</option> <option value="as">Assamese</option>
<option value="tt">Tatar</option> <option value="az">Azerbaijani</option>
<option value="haw">Hawaiian</option>
<option value="ln">Lingala</option>
<option value="ha">Hausa</option>
<option value="ba">Bashkir</option> <option value="ba">Bashkir</option>
<option value="eu">Basque</option>
<option value="be">Belarusian</option>
<option value="bn">Bengali</option>
<option value="bs">Bosnian</option>
<option value="br">Breton</option>
<option value="bg">Bulgarian</option>
<option value="ca">Catalan</option>
<option value="zh">Chinese</option>
<option value="hr">Croatian</option>
<option value="cs">Czech</option>
<option value="da">Danish</option>
<option value="nl">Dutch</option>
<option value="en">English</option>
<option value="et">Estonian</option>
<option value="fo">Faroese</option>
<option value="fi">Finnish</option>
<option value="fr">French</option>
<option value="gl">Galician</option>
<option value="ka">Georgian</option>
<option value="de">German</option>
<option value="el">Greek</option>
<option value="gu">Gujarati</option>
<option value="ht">Haitian Creole</option>
<option value="ha">Hausa</option>
<option value="haw">Hawaiian</option>
<option value="he">Hebrew</option>
<option value="hi">Hindi</option>
<option value="hu">Hungarian</option>
<option value="is">Icelandic</option>
<option value="id">Indonesian</option>
<option value="it">Italian</option>
<option value="ja">Japanese</option>
<option value="jw">Javanese</option> <option value="jw">Javanese</option>
<option value="kn">Kannada</option>
<option value="kk">Kazakh</option>
<option value="km">Khmer</option>
<option value="ko">Korean</option>
<option value="lo">Lao</option>
<option value="la">Latin</option>
<option value="lv">Latvian</option>
<option value="ln">Lingala</option>
<option value="lt">Lithuanian</option>
<option value="lb">Luxembourgish</option>
<option value="mk">Macedonian</option>
<option value="mg">Malagasy</option>
<option value="ms">Malay</option>
<option value="ml">Malayalam</option>
<option value="mt">Maltese</option>
<option value="mi">Maori</option>
<option value="mr">Marathi</option>
<option value="mn">Mongolian</option>
<option value="my">Myanmar</option>
<option value="ne">Nepali</option>
<option value="no">Norwegian</option>
<option value="nn">Nynorsk</option>
<option value="oc">Occitan</option>
<option value="ps">Pashto</option>
<option value="fa">Persian</option>
<option value="pl">Polish</option>
<option value="pt">Portuguese</option>
<option value="pa">Punjabi</option>
<option value="ro">Romanian</option>
<option value="ru">Russian</option>
<option value="sa">Sanskrit</option>
<option value="sr">Serbian</option>
<option value="sn">Shona</option>
<option value="sd">Sindhi</option>
<option value="si">Sinhala</option>
<option value="sk">Slovak</option>
<option value="sl">Slovenian</option>
<option value="so">Somali</option>
<option value="es">Spanish</option>
<option value="su">Sundanese</option> <option value="su">Sundanese</option>
<option value="sw">Swahili</option>
<option value="sv">Swedish</option>
<option value="tl">Tagalog</option>
<option value="tg">Tajik</option>
<option value="ta">Tamil</option>
<option value="tt">Tatar</option>
<option value="te">Telugu</option>
<option value="th">Thai</option>
<option value="bo">Tibetan</option>
<option value="tr">Turkish</option>
<option value="tk">Turkmen</option>
<option value="uk">Ukrainian</option>
<option value="ur">Urdu</option>
<option value="uz">Uzbek</option>
<option value="vi">Vietnamese</option>
<option value="cy">Welsh</option>
<option value="yi">Yiddish</option>
<option value="yo">Yoruba</option>
</select> </select>
</div> </div>
<div class="dropdown-container"> <div class="dropdown-container">
@@ -134,11 +131,15 @@
<div class="dropdown-container"> <div class="dropdown-container">
<label for="modelSizeDropdown">Select Model Size:</label> <label for="modelSizeDropdown">Select Model Size:</label>
<select id="modelSizeDropdown"> <select id="modelSizeDropdown">
<option value="">Select Task</option> <option value="">Select model</option>
<option value="tiny">Tiny</option> <option value="tiny">Tiny </option>
<option value="tiny.en">Tiny (English-only)</option>
<option value="base">Base</option> <option value="base">Base</option>
<option value="base.en">Base (English-only)</option>
<option value="small" selected>Small</option> <option value="small" selected>Small</option>
<option value="small.en">Small (English-only)</option>
<option value="medium">Medium</option> <option value="medium">Medium</option>
<option value="medium.en">Medium (English-only)</option>
<option value="large-v2">Large-v2</option> <option value="large-v2">Large-v2</option>
<option value="large-v3">Large-v3</option> <option value="large-v3">Large-v3</option>
</select> </select>
Audio-Transcription-Chrome/popup.js
+1 -24
View File
@@ -4,7 +4,6 @@ document.addEventListener("DOMContentLoaded", function () {
const stopButton = document.getElementById("stopCapture"); const stopButton = document.getElementById("stopCapture");
const useServerCheckbox = document.getElementById("useServerCheckbox"); const useServerCheckbox = document.getElementById("useServerCheckbox");
const useMultilingualCheckbox = document.getElementById('useMultilingualCheckbox');
const languageDropdown = document.getElementById('languageDropdown'); const languageDropdown = document.getElementById('languageDropdown');
const taskDropdown = document.getElementById('taskDropdown'); const taskDropdown = document.getElementById('taskDropdown');
const modelSizeDropdown = document.getElementById('modelSizeDropdown'); const modelSizeDropdown = document.getElementById('modelSizeDropdown');
@@ -32,14 +31,6 @@ document.addEventListener("DOMContentLoaded", function () {
} }
}); });
chrome.storage.local.get("useMultilingualModelState", ({ useMultilingualModelState }) => {
if (useMultilingualModelState !== undefined) {
useMultilingualCheckbox.checked = useMultilingualModelState;
languageDropdown.disabled = !useMultilingualModelState;
taskDropdown.disabled = !useMultilingualModelState;
}
});
chrome.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => { chrome.storage.local.get("selectedLanguage", ({ selectedLanguage: storedLanguage }) => {
if (storedLanguage !== undefined) { if (storedLanguage !== undefined) {
languageDropdown.value = storedLanguage; languageDropdown.value = storedLanguage;
@@ -86,7 +77,6 @@ document.addEventListener("DOMContentLoaded", function () {
tabId: currentTab.id, tabId: currentTab.id,
host: host, host: host,
port: port, port: port,
useMultilingual: useMultilingualCheckbox.checked,
language: selectedLanguage, language: selectedLanguage,
task: selectedTask, task: selectedTask,
modelSize: selectedModelSize modelSize: selectedModelSize
@@ -129,9 +119,8 @@ document.addEventListener("DOMContentLoaded", function () {
startButton.disabled = isCapturing; startButton.disabled = isCapturing;
stopButton.disabled = !isCapturing; stopButton.disabled = !isCapturing;
useServerCheckbox.disabled = isCapturing; useServerCheckbox.disabled = isCapturing;
useMultilingualCheckbox.disabled = isCapturing;
modelSizeDropdown.disabled = isCapturing; modelSizeDropdown.disabled = isCapturing;
taskDropdown.disabled = isCapturing;
startButton.classList.toggle("disabled", isCapturing); startButton.classList.toggle("disabled", isCapturing);
stopButton.classList.toggle("disabled", !isCapturing); stopButton.classList.toggle("disabled", !isCapturing);
} }
@@ -142,18 +131,6 @@ document.addEventListener("DOMContentLoaded", function () {
chrome.storage.local.set({ useServerState }); chrome.storage.local.set({ useServerState });
}); });
useMultilingualCheckbox.addEventListener('change', function() {
const useMultilingualModelState = useMultilingualCheckbox.checked;
if (useMultilingualModelState) {
languageDropdown.disabled = false;
taskDropdown.disabled = false;
} else {
languageDropdown.disabled = true;
taskDropdown.disabled = true;
}
chrome.storage.local.set({ useMultilingualModelState });
});
languageDropdown.addEventListener('change', function() { languageDropdown.addEventListener('change', function() {
if (languageDropdown.value === "") { if (languageDropdown.value === "") {
selectedLanguage = null; selectedLanguage = null;
whisper_live/client.py
+5 -13
View File
@@ -76,7 +76,6 @@ class Client:
self, self,
host=None, host=None,
port=None, port=None,
is_multilingual=False,
lang=None, lang=None,
translate=False, translate=False,
model="small", model="small",
@@ -92,8 +91,7 @@ class Client:
Args: Args:
host (str): The hostname or IP address of the server. host (str): The hostname or IP address of the server.
port (int): The port number for the WebSocket server. port (int): The port number for the WebSocket server.
is_multilingual (bool, optional): Specifies if multilingual transcription is enabled. Default is False. lang (str, optional): The selected language for transcription. Default is None.
lang (str, optional): The selected language for transcription when multilingual is disabled. Default is None.
translate (bool, optional): Specifies if the task is translation. Default is False. translate (bool, optional): Specifies if the task is translation. Default is False.
""" """
self.chunk = 4096 self.chunk = 4096
@@ -102,14 +100,11 @@ class Client:
self.rate = 16000 self.rate = 16000
self.record_seconds = 60000 self.record_seconds = 60000
self.recording = False self.recording = False
self.multilingual = False
self.language = None
self.task = "transcribe" self.task = "transcribe"
self.uid = str(uuid.uuid4()) self.uid = str(uuid.uuid4())
self.waiting = False self.waiting = False
self.last_response_recieved = None self.last_response_recieved = None
self.disconnect_if_no_response_for = 15 self.disconnect_if_no_response_for = 15
self.multilingual = is_multilingual
self.language = lang self.language = lang
self.model = model self.model = model
self.server_error = False self.server_error = False
@@ -247,7 +242,7 @@ class Client:
""" """
Callback function called when the WebSocket connection is successfully opened. Callback function called when the WebSocket connection is successfully opened.
Sends an initial configuration message to the server, including client UID, multilingual mode, Sends an initial configuration message to the server, including client UID,
language selection, and task type. language selection, and task type.
Args: Args:
@@ -261,7 +256,6 @@ class Client:
json.dumps( json.dumps(
{ {
"uid": self.uid, "uid": self.uid,
"multilingual": self.multilingual,
"language": self.language, "language": self.language,
"task": self.task, "task": self.task,
"model": self.model, "model": self.model,
@@ -548,8 +542,7 @@ class TranscriptionClient:
Args: Args:
host (str): The hostname or IP address of the server. host (str): The hostname or IP address of the server.
port (int): The port number to connect to on the server. port (int): The port number to connect to on the server.
is_multilingual (bool, optional): Indicates whether the transcription should support multiple languages (default is False). lang (str, optional): The primary language for transcription. Default is None, which defaults to English ('en').
lang (str, optional): The primary language for transcription (used if `is_multilingual` is False). Default is None, which defaults to English ('en').
translate (bool, optional): Indicates whether translation tasks are required (default is False). translate (bool, optional): Indicates whether translation tasks are required (default is False).
Attributes: Attributes:
@@ -558,19 +551,18 @@ class TranscriptionClient:
Example: Example:
To create a TranscriptionClient and start transcription on microphone audio: To create a TranscriptionClient and start transcription on microphone audio:
```python ```python
transcription_client = TranscriptionClient(host="localhost", port=9090, is_multilingual=True) transcription_client = TranscriptionClient(host="localhost", port=9090)
transcription_client() transcription_client()
``` ```
""" """
def __init__(self, def __init__(self,
host, host,
port, port,
is_multilingual=False,
lang=None, lang=None,
translate=False, translate=False,
model="small", model="small",
): ):
self.client = Client(host, port, is_multilingual, lang, translate, model) self.client = Client(host, port, lang, translate, model)
def __call__(self, audio=None, hls_url=None): def __call__(self, audio=None, hls_url=None):
""" """
whisper_live/server.py
+11 -18
View File
@@ -136,6 +136,7 @@ class TranscriptionServer:
) )
logging.info(f"Running TensorRT backend.") logging.info(f"Running TensorRT backend.")
except Exception as e: except Exception as e:
self.client_uid = options["uid"]
websocket.send( websocket.send(
json.dumps( json.dumps(
{ {
@@ -154,7 +155,6 @@ class TranscriptionServer:
options["model"] = faster_whisper_custom_model_path options["model"] = faster_whisper_custom_model_path
client = ServeClientFasterWhisper( client = ServeClientFasterWhisper(
websocket, websocket,
multilingual=options["multilingual"],
language=options["language"], language=options["language"],
task=options["task"], task=options["task"],
client_uid=options["uid"], client_uid=options["uid"],
@@ -558,7 +558,6 @@ class ServeClientFasterWhisper(ServeClientBase):
websocket, websocket,
task="transcribe", task="transcribe",
device=None, device=None,
multilingual=False,
language=None, language=None,
client_uid=None, client_uid=None,
model="small.en", model="small.en",
@@ -575,7 +574,6 @@ class ServeClientFasterWhisper(ServeClientBase):
websocket (WebSocket): The WebSocket connection for the client. websocket (WebSocket): The WebSocket connection for the client.
task (str, optional): The task type, e.g., "transcribe." Defaults to "transcribe". task (str, optional): The task type, e.g., "transcribe." Defaults to "transcribe".
device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None. device (str, optional): The device type for Whisper, "cuda" or "cpu". Defaults to None.
multilingual (bool, optional): Whether the client supports multilingual transcription. Defaults to False.
language (str, optional): The language for transcription. Defaults to None. language (str, optional): The language for transcription. Defaults to None.
client_uid (str, optional): A unique identifier for the client. Defaults to None. client_uid (str, optional): A unique identifier for the client. Defaults to None.
@@ -585,13 +583,11 @@ class ServeClientFasterWhisper(ServeClientBase):
"tiny", "tiny.en", "base", "base.en", "small", "small.en", "tiny", "tiny.en", "base", "base.en", "small", "small.en",
"medium", "medium.en", "large-v2", "large-v3", "medium", "medium.en", "large-v2", "large-v3",
] ]
self.multilingual = multilingual
if not os.path.exists(model): if not os.path.exists(model):
self.model_size_or_path = self.get_model_size(model) self.model_size_or_path = self.check_valid_model(model)
else: else:
self.model_size_or_path = model self.model_size_or_path = model
self.language = language if self.multilingual else "en" self.language = "en" if self.model_size_or_path.endswith("en") else language
self.task = task self.task = task
self.initial_prompt = initial_prompt self.initial_prompt = initial_prompt
self.vad_parameters = vad_parameters or {"threshold": 0.5} self.vad_parameters = vad_parameters or {"threshold": 0.5}
@@ -622,9 +618,15 @@ class ServeClientFasterWhisper(ServeClientBase):
) )
) )
def get_model_size(self, model_size): def check_valid_model(self, model_size):
""" """
Returns the whisper model size based on multilingual. Check if it's a valid whisper model size.
Args:
model_size (str): The name of the model size to check.
Returns:
str: The model size if valid, None otherwise.
""" """
if model_size not in self.model_sizes: if model_size not in self.model_sizes:
self.websocket.send( self.websocket.send(
@@ -637,15 +639,6 @@ class ServeClientFasterWhisper(ServeClientBase):
) )
) )
return None return None
if model_size.endswith("en") and self.multilingual:
logging.info(f"Setting multilingual to false with {model_size} which is english only model.")
self.multilingual = False
if not model_size.endswith("en") and not self.multilingual:
logging.info(f"Setting multilingual to true with multilingual model {model_size}.")
self.multilingual = True
return model_size return model_size
def speech_to_text(self): def speech_to_text(self):