Offline Inference With Default Generation Config#
Source: examples/offline_inference_with_default_generation_config.py.
1from vllm import LLM
2
3# Sample prompts.
4prompts = [
5 "Hello, my name is",
6 "The president of the United States is",
7 "The capital of France is",
8 "The future of AI is",
9]
10
11# Create an LLM with built-in default generation config.
12# The generation config is set to None by default to keep
13# the behavior consistent with the previous version.
14# If you want to use the default generation config from the model,
15# you should set the generation_config to "auto".
16llm = LLM(model="Qwen/Qwen2.5-0.5B-Instruct", generation_config="auto")
17
18# Load the default sampling parameters from the model.
19sampling_params = llm.get_default_sampling_params()
20# Modify the sampling parameters if needed.
21sampling_params.temperature = 0.5
22
23# Generate texts from the prompts. The output is a list of RequestOutput objects
24# that contain the prompt, generated text, and other information.
25outputs = llm.generate(prompts, sampling_params)
26# Print the outputs.
27for output in outputs:
28 prompt = output.prompt
29 generated_text = output.outputs[0].text
30 print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")