{"model_id": "Baichuan2-13B-Chat", "conv_template_name": "baichuan2-chat"}

run

python3 -m vllm.entrypoints.openai.api_server --model /data/model --served-model-name model --trust-remote-code --quantization ifq --use-v2-block-manager --use-lookahead --num-speculative-tokens 6 

run

curl -H "content-type: application/json" http://localhost:8501/v1/chat/completions -d '{"messages":[{"role": "user", "content": "hello"}], "temperature": 0.0}'

Hello! What can I do for you?

curl -H "content-type: application/json" http://localhost:8501/v1/chat/completions -d '{"messages":[{"role": "user", "content": "hello"}], "temperature": 0.0, "stream": true}'

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "you"}, "finish_reason": null}]}

Okay.

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "what are my"}, "finish_reason": null}]}

{"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "Can help"}, "finish_reason": null}]}

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {"content": "To yours"}, "finish_reason": null}]}

?

data: {"id": "chatcmpl-3fYW8fqN3YYMJiebiZgpzZ", "object": "chat.completion.chunk", "created": 1698290857, "model": "baichuan-13b-chat", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}

data: [DONE]

import os
from openai import OpenAI

# Example 1: Container local access address
os.environ["OPENAI_BASE_URL"] = "http://127.0.0.1:8501/v1"
# Example 2: Public network access address, which can be obtained through the service invocation page (add v1 after the address when using openai sdk)
os.environ["OPENAI_BASE_URL"] = "https://service-********.sh.tencentapigw.com:443/tione/v1"
# If the --api-keys parameter is set when the service starts or the OPENAI_API_KEYS environment variable enables authentication, OPENAI_API_KEY needs to be set to any one of the available api_keys; otherwise, it can be filled in arbitrarily.
os.environ["OPENAI_API_KEY"] = "EMPTY"

client = OpenAI()

# Non-streaming request example:
print("----- standard request -----")
completion = client.chat.completions.create(
    model="model",
    messages=[
        {
            "role": "user",
            Hello
        },
    ],
    temperature=0.7,
    top_p=1.0, 
    max_tokens=128,
)
print(completion.choices[0].message.content)

# Streaming request example:
print("----- streaming request -----")
stream = client.chat.completions.create(
    model="model",
    messages=[
        {
            "role": "user",
            Hello
        },
    ],
    temperature=0.7,
    top_p=1.0,
    max_tokens=128,
    stream=True,
)
for chunk in stream:
    if not chunk.choices or not chunk.choices[0].delta.content:
        continue
    print(chunk.choices[0].delta.content, end="")
print()


import argparse
import requests
import json


def chat(messages):
    data = {
        "messages": messages,
        "temperature": args.temperature,
        "max_tokens": args.max_tokens,
        "top_p": args.top_p,
        "stream": True,  # Enable streaming output
    }
    header = {
        "Content-Type": "application/json",
    }
    if args.token:
        header["Authorization"] = f"Bearer {args.token}"

    response = requests.post(f"{args.server}/v1/chat/completions", json=data, headers=header, stream=True)  # Set stream=True to get real-time data stream
    if response.status_code != 200:
        print(response.json())
        exit()

    result = ""
    print("Assistant: ", end = "", flush = True)
    for part in response.iter_lines():
        if part:
            if "content" in part.decode("utf-8"):
                content = json.loads(part.decode("utf-8")[5:])["choices"][0]["delta"]["content"]  # String filtering. Convert data into json format and then extract text.
                result += content
                print(content, end = "", flush = True)
    print("\\n")

    return result


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Chat CLI Demo.")    
    parser.add_argument("--server", type=str, default="http://127.0.0.1:8501")
    parser.add_argument("--max-tokens", type=int, default=512)
    parser.add_argument("--temperature", type=float, default=0.7)
    parser.add_argument("--top_p", type=float, default=1.0)
    parser.add_argument("--system", type=str, default=None)
    parser.add_argument("--token", type=str, default=None)
    args = parser.parse_args()

    messages = []
    if args.system:
        messages.append({"role": "system", "content": args.system})

    while True:
        user_input = input("User: ")
        messages.append({"role": "user", "content": user_input})
        response = chat(messages)
        messages.append({"role": "assistant", "content": response})