From ab17c4dbc6cf79c792ff4aedf4cc90461291d56c Mon Sep 17 00:00:00 2001
From: Andreas Peldszus <andreas.peldszus@retresco.de>
Date: Wed, 5 Jun 2024 09:44:04 +0200
Subject: [PATCH] Make single model mode the default, update readme

---
 README.md     | 14 +++++---------
 run_server.py |  5 +++--
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index 2a363c3..47f8f6d 100644
--- a/README.md
+++ b/README.md
@@ -62,16 +62,12 @@ python3 run_server.py --port 9090 \
 ```
 
 #### Single model mode
-By default, the server will instantiate a new whisper model for every client connection. I.e. for 4 concurrent connections, you need to have (V)RAM for 4x the model. Also, the connection will have to wait until the model is loaded, which might take a few seconds.
+By default, when running the server without specifying a model, the server will instantiate a new whisper model for every client connection. This has the advantage, that the server can use different model sizes, based on the client's requested model size. On the other hand, it also means you have to wait for the model to be loaded upon client connection and you will have increased (V)RAM usage.
+
+When serving a custom TensorRT model using the `-trt` or a custom faster_whisper model using the `-fw` option, the server will instead make use of the `--single_model true` option: It will only instantiate the custom model once and reuse that for all client connections.
+
+If you don't want this, set `--single_model false`.
 
-If you use a TensorRT or a custom faster_whisper model, you also set the server to instantiate the model only once and use the single on for all client connections, using the `--single-model` or `-sm` option:
-```bash
-python3 run_server.py -p 9090 \
-                      -b tensorrt \
-                      -trt /home/TensorRT-LLM/examples/whisper/whisper_large_v3 \
-                      -m \
-                      --single_model
-```
 
 ### Running the Client
 - Initializing the client with below parameters:
diff --git a/run_server.py b/run_server.py
index b60c3bd..ed038d2 100644
--- a/run_server.py
+++ b/run_server.py
@@ -26,8 +26,9 @@ if __name__ == "__main__":
                         default=1,
                         help="Number of threads to use for OpenMP")
     parser.add_argument('--single_model', '-sm',
-                        action="store_true",
-                        help='Set to true if only one (custom) model instance should be served.')
+                        type=bool,
+                        default=True,
+                        help='Set to false if every connection should instantiate its own model. Only relevant for custom model, passed using -trt or -fw.')
     args = parser.parse_args()
 
     if args.backend == "tensorrt":