Offline Inference Scoring#
Source: examples/offline_inference_scoring.py.
1from vllm import LLM
2
3# Sample prompts.
4text_1 = "What is the capital of France?"
5texts_2 = [
6 "The capital of Brazil is Brasilia.", "The capital of France is Paris."
7]
8
9# Create an LLM.
10# You should pass task="score" for cross-encoder models
11model = LLM(
12 model="BAAI/bge-reranker-v2-m3",
13 task="score",
14 enforce_eager=True,
15)
16
17# Generate scores. The output is a list of ScoringRequestOutputs.
18outputs = model.score(text_1, texts_2)
19
20# Print the outputs.
21for text_2, output in zip(texts_2, outputs):
22 score = output.outputs.score
23 print(f"Pair: {[text_1, text_2]!r} | Score: {score}")