{"model_id": "Baichuan2-13B-Chat", "conv_template_name": "baichuan2-chat"}

run

python3 -m vllm.entrypoints.openai.api_server --model /data/model --served-model-name model --trust-remote-code --quantization ifq --use-v2-block-manager --use-lookahead --num-speculative-tokens 6 

run

curl -H "content-type: application/json" http://localhost:8501/v1/chat/completions -d '{"messages":[{"role": "user", "content": "你好"}], "temperature": 0.0}'

{"id":"chatcmpl-4aeRgYwnaYe4RzmmcyKtYs","object":"chat.completion","created":1698291242,"model":"baichuan-13b-chat","choices":[{"index":0,"message":{"role":"assistant","content":"你好！有什么我能帮到你的吗？"},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"total_tokens":16,"completion_tokens":12}}

curl -H "content-type: application/json" http://localhost:8501/v1/chat/completions -d '{"messages":[{"role": "user", "content": "你好"}], "temperature": 0.0, "stream": true}'

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "你"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "好！"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "有什么我"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "能帮"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "到你的"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "吗？"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "object": "chat.completion.chunk", "created": 1698290857, "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}

data: [DONE]

import os
from openai import OpenAI

# 示例1：容器本地访问地址
os.environ["OPENAI_BASE_URL"] = "http://127.0.0.1:8501/v1"
# 示例2：外网访问地址，可以通过服务调用页面获取（使用openai sdk时，地址后面加上v1）
os.environ["OPENAI_BASE_URL"] = "https://service-********.sh.tencentapigw.com:443/tione/v1"
# 若服务启动设置了 --api-keys 参数或 OPENAI_API_KEYS 环境变量开启了鉴权，OPENAI_API_KEY 需要设置为任意一个可用的 api_key；否则可以任意填写
os.environ["OPENAI_API_KEY"] = "EMPTY"

client = OpenAI()

# 非流式请求示例：
print("----- standard request -----")
completion = client.chat.completions.create(
    model="model",
    messages=[
        {
            "role": "user",
            "content": "你好",
        },
    ],
    temperature=0.7,
    top_p=1.0, 
    max_tokens=128,
)
print(completion.choices[0].message.content)

# 流式请求示例:
print("----- streaming request -----")
stream = client.chat.completions.create(
    model="model",
    messages=[
        {
            "role": "user",
            "content": "你好",
        },
    ],
    temperature=0.7,
    top_p=1.0,
    max_tokens=128,
    stream=True,
)
for chunk in stream:
    if not chunk.choices or not chunk.choices[0].delta.content:
        continue
    print(chunk.choices[0].delta.content, end="")
print()


import argparse
import requests
import json


def chat(messages):
    data = {
        "messages": messages,
        "temperature": args.temperature,
        "max_tokens": args.max_tokens,
        "top_p": args.top_p,
        "stream": True,  # 开启流式输出
    }
    header = {
        "Content-Type": "application/json",
    }
    if args.token:
        header["Authorization"] = f"Bearer {args.token}"

    response = requests.post(f"{args.server}/v1/chat/completions", json=data, headers=header, stream=True)  # 设置 stream=True 参数获取实时数据流
    if response.status_code != 200:
        print(response.json())
        exit()

    result = ""
    print("Assistant: ", end = "", flush = True)
    for part in response.iter_lines():
        if part:
            if "content" in part.decode("utf-8"):
                content = json.loads(part.decode("utf-8")[5:])["choices"][0]["delta"]["content"]  # 字符串过滤 data: 之后转成 json 格式再取文本
                result += content
                print(content, end = "", flush = True)
    print("\\n")

    return result


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Chat CLI Demo.")    
    parser.add_argument("--server", type=str, default="http://127.0.0.1:8501")
    parser.add_argument("--max-tokens", type=int, default=512)
    parser.add_argument("--temperature", type=float, default=0.7)
    parser.add_argument("--top_p", type=float, default=1.0)
    parser.add_argument("--system", type=str, default=None)
    parser.add_argument("--token", type=str, default=None)
    args = parser.parse_args()

    messages = []
    if args.system:
        messages.append({"role": "system", "content": args.system})

    while True:
        user_input = input("User: ")
        messages.append({"role": "user", "content": user_input})
        response = chat(messages)
        messages.append({"role": "assistant", "content": response})