Merge remote-tracking branch 'upstream/main' into add_client_queue

2023-08-08 17:34:00 +08:00
parent 19b9d8b03d 36e1f03f37
commit 467589db3c
4 changed files with 52 additions and 4 deletions
@@ -149,10 +149,11 @@ class ServeClient:
        self.frames = b""
        self.language = language if multilingual else "en"
        self.task = task
+        device = "cuda" if torch.cuda.is_available() else "cpu"
        self.transcriber = WhisperModel(
            "small" if multilingual else "small.en", 
-            device=device if device else "cuda",
-            compute_type="float16", 
+            device=device,
+            compute_type="int8" if device=="cpu" else "float16", 
            local_files_only=False,
        )