Spaces:
Runtime error
Runtime error
Update run.sh
Browse files
run.sh
CHANGED
|
@@ -7,6 +7,8 @@ pip install -v -r requirements.txt --extra-index-url https://download.pytorch.or
|
|
| 7 |
git clone https://github.com/vllm-project/vllm.git
|
| 8 |
cd vllm && VLLM_TARGET_DEVICE=cpu python setup.py install && cd ..
|
| 9 |
git clone https://huggingface.co/Moses25/Mistral-7B-Instruct-32K-AWQ
|
|
|
|
|
|
|
| 10 |
python -m vllm.entrypoints.openai.api_server --model=Mistral-7B-Instruct-32K-AWQ \
|
| 11 |
--trust-remote-code --host 0.0.0.0 --port 7777 \
|
| 12 |
--gpu-memory-utilization 0.8 \
|
|
|
|
| 7 |
git clone https://github.com/vllm-project/vllm.git
|
| 8 |
cd vllm && VLLM_TARGET_DEVICE=cpu python setup.py install && cd ..
|
| 9 |
git clone https://huggingface.co/Moses25/Mistral-7B-Instruct-32K-AWQ
|
| 10 |
+
|
| 11 |
+
set VLLM_CPU_KVCACHE_SPACE=15
|
| 12 |
python -m vllm.entrypoints.openai.api_server --model=Mistral-7B-Instruct-32K-AWQ \
|
| 13 |
--trust-remote-code --host 0.0.0.0 --port 7777 \
|
| 14 |
--gpu-memory-utilization 0.8 \
|