OpenAI Audio API Client#

Source vllm-project/vllm.

 1"""An example showing how to use vLLM to serve VLMs.
 2
 3Launch the vLLM server with the following command:
 4vllm serve fixie-ai/ultravox-v0_3
 5"""
 6import base64
 7
 8import requests
 9from openai import OpenAI
10
11from vllm.assets.audio import AudioAsset
12
13# Modify OpenAI's API key and API base to use vLLM's API server.
14openai_api_key = "EMPTY"
15openai_api_base = "http://localhost:8000/v1"
16
17client = OpenAI(
18    # defaults to os.environ.get("OPENAI_API_KEY")
19    api_key=openai_api_key,
20    base_url=openai_api_base,
21)
22
23models = client.models.list()
24model = models.data[0].id
25
26# Any format supported by librosa is supported
27audio_url = AudioAsset("winning_call").url
28
29# Use audio url in the payload
30chat_completion_from_url = client.chat.completions.create(
31    messages=[{
32        "role":
33        "user",
34        "content": [
35            {
36                "type": "text",
37                "text": "What's in this audio?"
38            },
39            {
40                "type": "audio_url",
41                "audio_url": {
42                    "url": audio_url
43                },
44            },
45        ],
46    }],
47    model=model,
48    max_tokens=64,
49)
50
51result = chat_completion_from_url.choices[0].message.content
52print(f"Chat completion output:{result}")
53
54
55# Use base64 encoded audio in the payload
56def encode_audio_base64_from_url(audio_url: str) -> str:
57    """Encode an audio retrieved from a remote url to base64 format."""
58
59    with requests.get(audio_url) as response:
60        response.raise_for_status()
61        result = base64.b64encode(response.content).decode('utf-8')
62
63    return result
64
65
66audio_base64 = encode_audio_base64_from_url(audio_url=audio_url)
67chat_completion_from_base64 = client.chat.completions.create(
68    messages=[{
69        "role":
70        "user",
71        "content": [
72            {
73                "type": "text",
74                "text": "What's in this audio?"
75            },
76            {
77                "type": "audio_url",
78                "audio_url": {
79                    # Any format supported by librosa is supported
80                    "url": f"data:audio/ogg;base64,{audio_base64}"
81                },
82            },
83        ],
84    }],
85    model=model,
86    max_tokens=64,
87)
88
89result = chat_completion_from_base64.choices[0].message.content
90print(f"Chat completion output:{result}")