Skip to content

vllm.entrypoints.openai.serving_classification

logger module-attribute 

logger = init_logger(__name__)

ClassificationMixin

Bases: OpenAIServing

Source code in vllm/entrypoints/openai/serving_classification.py 
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
class ClassificationMixin(OpenAIServing):

    @override
    async def _preprocess(
        self,
        ctx: ServeContext,
    ) -> Optional[ErrorResponse]:
        """
        Process classification inputs: tokenize text, resolve adapters,
        and prepare model-specific inputs.
        """
        ctx = cast(ClassificationServeContext, ctx)
        if isinstance(ctx.request.input, str) and not ctx.request.input:
            return self.create_error_response(
                "Input cannot be empty for classification",
                status_code=HTTPStatus.BAD_REQUEST,
            )

        if isinstance(ctx.request.input, list) and len(ctx.request.input) == 0:
            return None

        try:
            ctx.lora_request = self._maybe_get_adapters(ctx.request)

            ctx.tokenizer = await self.engine_client.get_tokenizer(
                ctx.lora_request)

            (
                ctx.request_prompts,
                ctx.engine_prompts,
            ) = await self._preprocess_completion(
                ctx.request,
                ctx.tokenizer,
                ctx.request.input,
                truncate_prompt_tokens=ctx.request.truncate_prompt_tokens,
            )

            return None

        except (ValueError, TypeError) as e:
            logger.exception("Error in preprocessing prompt inputs")
            return self.create_error_response(str(e))

    @override
    def _build_response(
        self,
        ctx: ServeContext,
    ) -> Union[ClassificationResponse, ErrorResponse]:
        """
        Convert model outputs to a formatted classification response
        with probabilities and labels.
        """
        ctx = cast(ClassificationServeContext, ctx)
        items: list[ClassificationData] = []
        num_prompt_tokens = 0

        final_res_batch_checked = cast(list[PoolingRequestOutput],
                                       ctx.final_res_batch)

        for idx, final_res in enumerate(final_res_batch_checked):
            classify_res = ClassificationOutput.from_base(final_res.outputs)

            probs = classify_res.probs
            predicted_index = int(np.argmax(probs))
            label = getattr(self.model_config.hf_config, "id2label",
                            {}).get(predicted_index)

            item = ClassificationData(
                index=idx,
                label=label,
                probs=probs,
                num_classes=len(probs),
            )

            items.append(item)
            prompt_token_ids = final_res.prompt_token_ids
            num_prompt_tokens += len(prompt_token_ids)

        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            total_tokens=num_prompt_tokens,
        )

        return ClassificationResponse(
            id=ctx.request_id,
            created=ctx.created_time,
            model=ctx.model_name,
            data=items,
            usage=usage,
        )

_build_response 

_build_response(
    ctx: ServeContext,
) -> Union[ClassificationResponse, ErrorResponse]

Convert model outputs to a formatted classification response with probabilities and labels.

Source code in vllm/entrypoints/openai/serving_classification.py 
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@override
def _build_response(
    self,
    ctx: ServeContext,
) -> Union[ClassificationResponse, ErrorResponse]:
    """
    Convert model outputs to a formatted classification response
    with probabilities and labels.
    """
    ctx = cast(ClassificationServeContext, ctx)
    items: list[ClassificationData] = []
    num_prompt_tokens = 0

    final_res_batch_checked = cast(list[PoolingRequestOutput],
                                   ctx.final_res_batch)

    for idx, final_res in enumerate(final_res_batch_checked):
        classify_res = ClassificationOutput.from_base(final_res.outputs)

        probs = classify_res.probs
        predicted_index = int(np.argmax(probs))
        label = getattr(self.model_config.hf_config, "id2label",
                        {}).get(predicted_index)

        item = ClassificationData(
            index=idx,
            label=label,
            probs=probs,
            num_classes=len(probs),
        )

        items.append(item)
        prompt_token_ids = final_res.prompt_token_ids
        num_prompt_tokens += len(prompt_token_ids)

    usage = UsageInfo(
        prompt_tokens=num_prompt_tokens,
        total_tokens=num_prompt_tokens,
    )

    return ClassificationResponse(
        id=ctx.request_id,
        created=ctx.created_time,
        model=ctx.model_name,
        data=items,
        usage=usage,
    )

_preprocess async 

_preprocess(ctx: ServeContext) -> Optional[ErrorResponse]

Process classification inputs: tokenize text, resolve adapters, and prepare model-specific inputs.

Source code in vllm/entrypoints/openai/serving_classification.py 
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
@override
async def _preprocess(
    self,
    ctx: ServeContext,
) -> Optional[ErrorResponse]:
    """
    Process classification inputs: tokenize text, resolve adapters,
    and prepare model-specific inputs.
    """
    ctx = cast(ClassificationServeContext, ctx)
    if isinstance(ctx.request.input, str) and not ctx.request.input:
        return self.create_error_response(
            "Input cannot be empty for classification",
            status_code=HTTPStatus.BAD_REQUEST,
        )

    if isinstance(ctx.request.input, list) and len(ctx.request.input) == 0:
        return None

    try:
        ctx.lora_request = self._maybe_get_adapters(ctx.request)

        ctx.tokenizer = await self.engine_client.get_tokenizer(
            ctx.lora_request)

        (
            ctx.request_prompts,
            ctx.engine_prompts,
        ) = await self._preprocess_completion(
            ctx.request,
            ctx.tokenizer,
            ctx.request.input,
            truncate_prompt_tokens=ctx.request.truncate_prompt_tokens,
        )

        return None

    except (ValueError, TypeError) as e:
        logger.exception("Error in preprocessing prompt inputs")
        return self.create_error_response(str(e))

ServingClassification

Bases: ClassificationMixin

Source code in vllm/entrypoints/openai/serving_classification.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
class ServingClassification(ClassificationMixin):
    request_id_prefix = "classify"

    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
        models: OpenAIServingModels,
        *,
        request_logger: Optional[RequestLogger],
    ) -> None:
        super().__init__(
            engine_client=engine_client,
            model_config=model_config,
            models=models,
            request_logger=request_logger,
        )

    async def create_classify(
        self,
        request: ClassificationRequest,
        raw_request: Request,
    ) -> Union[ClassificationResponse, ErrorResponse]:
        model_name = self._get_model_name(request.model)
        request_id = (f"{self.request_id_prefix}-"
                      f"{self._base_request_id(raw_request)}")

        ctx = ClassificationServeContext(
            request=request,
            raw_request=raw_request,
            model_name=model_name,
            request_id=request_id,
        )

        return await super().handle(ctx)  # type: ignore

    @override
    def _validate_request(
        self,
        ctx: ClassificationServeContext,
    ) -> Optional[ErrorResponse]:
        if error := super()._validate_request(ctx):
            return error

        ctx.truncate_prompt_tokens = ctx.request.truncate_prompt_tokens

        return None

    @override
    def _create_pooling_params(
        self,
        ctx: ClassificationServeContext,
    ) -> Union[PoolingParams, ErrorResponse]:
        pooling_params = super()._create_pooling_params(ctx)
        if isinstance(pooling_params, ErrorResponse):
            return pooling_params

        try:
            pooling_params.verify("classify", self.model_config)
        except ValueError as e:
            return self.create_error_response(str(e))

        return pooling_params

request_id_prefix class-attribute instance-attribute 

request_id_prefix = 'classify'

__init__ 

__init__(
    engine_client: EngineClient,
    model_config: ModelConfig,
    models: OpenAIServingModels,
    *,
    request_logger: Optional[RequestLogger],
) -> None
Source code in vllm/entrypoints/openai/serving_classification.py 
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def __init__(
    self,
    engine_client: EngineClient,
    model_config: ModelConfig,
    models: OpenAIServingModels,
    *,
    request_logger: Optional[RequestLogger],
) -> None:
    super().__init__(
        engine_client=engine_client,
        model_config=model_config,
        models=models,
        request_logger=request_logger,
    )

_create_pooling_params 

_create_pooling_params(
    ctx: ClassificationServeContext,
) -> Union[PoolingParams, ErrorResponse]
Source code in vllm/entrypoints/openai/serving_classification.py 
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
@override
def _create_pooling_params(
    self,
    ctx: ClassificationServeContext,
) -> Union[PoolingParams, ErrorResponse]:
    pooling_params = super()._create_pooling_params(ctx)
    if isinstance(pooling_params, ErrorResponse):
        return pooling_params

    try:
        pooling_params.verify("classify", self.model_config)
    except ValueError as e:
        return self.create_error_response(str(e))

    return pooling_params

_validate_request 

_validate_request(
    ctx: ClassificationServeContext,
) -> Optional[ErrorResponse]
Source code in vllm/entrypoints/openai/serving_classification.py 
158
159
160
161
162
163
164
165
166
167
168
@override
def _validate_request(
    self,
    ctx: ClassificationServeContext,
) -> Optional[ErrorResponse]:
    if error := super()._validate_request(ctx):
        return error

    ctx.truncate_prompt_tokens = ctx.request.truncate_prompt_tokens

    return None

create_classify async 

create_classify(
    request: ClassificationRequest, raw_request: Request
) -> Union[ClassificationResponse, ErrorResponse]
Source code in vllm/entrypoints/openai/serving_classification.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
async def create_classify(
    self,
    request: ClassificationRequest,
    raw_request: Request,
) -> Union[ClassificationResponse, ErrorResponse]:
    model_name = self._get_model_name(request.model)
    request_id = (f"{self.request_id_prefix}-"
                  f"{self._base_request_id(raw_request)}")

    ctx = ClassificationServeContext(
        request=request,
        raw_request=raw_request,
        model_name=model_name,
        request_id=request_id,
    )

    return await super().handle(ctx)  # type: ignore