Offline Inference With Prefix#

Source vllm-project/vllm.

 1from vllm import LLM, SamplingParams
 2
 3# NOTE: This is just a running example. For benchmarking purpose,
 4# please see benchmarks/benchmark_prefix_caching.py
 5
 6# Common prefix.
 7prefix = (
 8    "You are an expert school principal, skilled in effectively managing "
 9    "faculty and staff. Draft 10-15 questions for a potential first grade "
10    "Head Teacher for my K-12, all-girls', independent school that emphasizes "
11    "community, joyful discovery, and life-long learning. The candidate is "
12    "coming in for a first-round panel interview for a 8th grade Math "
13    "teaching role. They have 5 years of previous teaching experience "
14    "as an assistant teacher at a co-ed, public school with experience "
15    "in middle school math teaching. Based on these information, fulfill "
16    "the following paragraph: ")
17
18# Sample prompts.
19prompts = [
20    "Hello, my name is",
21    "The president of the United States is",
22    "The capital of France is",
23    "The future of AI is",
24]
25
26generating_prompts = [prefix + prompt for prompt in prompts]
27
28# Create a sampling params object.
29sampling_params = SamplingParams(temperature=0.0)
30
31# Create an LLM.
32regular_llm = LLM(model="facebook/opt-125m", gpu_memory_utilization=0.4)
33
34prefix_cached_llm = LLM(model="facebook/opt-125m",
35                        enable_prefix_caching=True,
36                        gpu_memory_utilization=0.4)
37print("Results without `enable_prefix_caching`")
38
39# Generate texts from the prompts. The output is a list of RequestOutput objects
40# that contain the prompt, generated text, and other information.
41outputs = regular_llm.generate(generating_prompts, sampling_params)
42
43regular_generated_texts = []
44# Print the outputs.
45for output in outputs:
46    prompt = output.prompt
47    generated_text = output.outputs[0].text
48    regular_generated_texts.append(generated_text)
49    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
50
51print("-" * 80)
52
53# Warmup so that the shared prompt's KV cache is computed.
54prefix_cached_llm.generate(generating_prompts[0], sampling_params)
55
56# Generate with prefix caching.
57outputs = prefix_cached_llm.generate(generating_prompts, sampling_params)
58
59print("Results with `enable_prefix_caching`")
60
61cached_generated_texts = []
62# Print the outputs. You should see the same outputs as before.
63for output in outputs:
64    prompt = output.prompt
65    generated_text = output.outputs[0].text
66    cached_generated_texts.append(generated_text)
67    print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
68
69print("-" * 80)
70
71# Compare the results and display the speedup
72generated_same = all([
73    regular_generated_texts[i] == cached_generated_texts[i]
74    for i in range(len(prompts))
75])
76print(f"Generated answers are the same: {generated_same}")