Update vLLM startup with extended context size and tool calling

README.md

@@ -15,9 +15,21 @@ pip install -r requirements.txt
 ```
 
 ### Running
-1. Start vLLM server:
+1. Start vLLM server with:
+- Extended context size (to 128k) through YaRN 
+- Hermes-style tools enabled
+
 ```bash
-pm2 start --name vllm 'vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 --dtype half --quantization gptq --gpu-memory-utilization 0.7 --port 8001'
+pm2 start --name vllm "vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 \
+  --dtype half \
+  --quantization gptq \
+  --gpu-memory-utilization 0.7 \
+  --dtype half --quantization gptq \
+  --rope-scaling '{\"rope_type\":\"yarn\",\"factor\":4.0,\"original_max_position_embeddings\":32768}' \
+  --max-model-len 131072 \
+  --enable-auto-tool-choice \
+  --tool-call-parser hermes \
+  --port 8001"
 ```
 
 2. Start the router:

scripts/autodeploy.sh

@@ -212,7 +212,16 @@ pm2 start ./qdrant --name qdrant --interpreter none -- --config-path config.yaml
 
 cd "$USER_HOME/$REPO_NAME" 
 pm2 start --name llm-router "uvicorn app.main:app --host 0.0.0.0 --port 8002"
-pm2 start --name vllm "vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 --dtype half --quantization gptq --gpu-memory-utilization 0.7 --port 8001"
+pm2 start --name vllm "vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 \
+  --dtype half \
+  --quantization gptq \
+  --gpu-memory-utilization 0.7 \
+  --dtype half --quantization gptq \
+  --rope-scaling '{\"rope_type\":\"yarn\",\"factor\":4.0,\"original_max_position_embeddings\":32768}' \
+  --max-model-len 131072 \
+  --enable-auto-tool-choice \
+  --tool-call-parser hermes \
+  --port 8001"
 
 # Save PM2 config
 pm2 save

scripts/standalone_deploy.sh

@@ -152,8 +152,13 @@ pip install .
 pm2 start --name 'llm-router' 'uvicorn app.main:app --host 0.0.0.0 --port 4444'
 
 # Start the model with vLLM
-pm2 start --name vllm 'vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 \
+pm2 start --name vllm "vllm serve Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4 \
   --dtype half \
   --quantization gptq \
   --gpu-memory-utilization 0.7 \
-  --port 8001'
+  --dtype half --quantization gptq \
+  --rope-scaling '{\"rope_type\":\"yarn\",\"factor\":4.0,\"original_max_position_embeddings\":32768}' \
+  --max-model-len 131072 \
+  --enable-auto-tool-choice \
+  --tool-call-parser hermes \
+  --port 8001"