pip3 install "fschat[model_worker,webui]"

pip3 install "fschat[all]"

git clone https://github.com/lm-sys/FastChat.gitcd FastChat

pip3 install --upgrade pip# enable PEP 660 supportpip3 install -e ".[model_worker,webui]"

python3 -m fastchat.serve.cli --model-path xxx

python3 -m fastchat.serve.cli --model-path xxx --num-gpus 2

python3 -m fastchat.serve.cli --model-path xxx --num-gpus 2 --max-gpu-memory 8GiB

python3 -m fastchat.serve.cli --model-path xxx --device cpu

python3 -m fastchat.serve.controller

pip install vllm

python3 -m fastchat.serve.vllm_worker --model-path xxx

CUDA_VISIBLE_DEVICES=0,1 python3 -m fastchat.serve.vllm_worker --model-path xxx --controller-address http://localhost:21001 --port 31000 --worker-address http://localhost:31000 --num-gpus 2

CUDA_VISIBLE_DEVICES=2,3 python3 -m fastchat.serve.vllm_worker --model-path xxx --controller-address http://localhost:21001 --port 31001 --worker-address http://localhost:31001 --num-gpus 2

CUDA_VISIBLE_DEVICES=4,5 python3 -m fastchat.serve.vllm_worker --model-path yyy --controller-address http://localhost:21001 --port 31001 --worker-address http://localhost:31001 --num-gpus 2

python3 -m fastchat.serve.openai_api_server --host localhost --port 8000

import openai
openai.api_key = "EMPTY"openai.base_url = "http://localhost:8000/v1/"
model = "vicuna-7b-v1.5"prompt = "Once upon a time"
# create a completioncompletion = openai.completions.create(model=model, prompt=prompt, max_tokens=64)# print the completionprint(prompt + completion.choices[0].text)
# create a chat completioncompletion = openai.chat.completions.create(model=model,messages=[{"role": "user", "content": "Hello! What is your name?"}])# print the completionprint(completion.choices[0].message.content)

curl http://localhost:8000/v1/chat/completions \-H "Content-Type: application/json" \-d '{"model": "vicuna-7b-v1.5","messages": [{"role": "user", "content": "Hello! What is your name?"}]}'

python3 -m fastchat.serve.controller

pip install vllm

python3 -m fastchat.serve.vllm_worker --model-path xxx

python3 -m fastchat.serve.gradio_web_server