Skip to content

vllm.parser.inkling

Inkling parser: typed content blocks parsed by a single state machine.

Inkling output format (every marker is a dedicated special token)::

<|message_model|><|content_thinking|>...reasoning...<|end_message|>
<|message_model|><|content_text|>...visible text...<|end_message|>
<|message_model|><|content_invoke_tool_json|>
    {"name":"get_weather","args":{"city":"SF"}}<|end_message|>

Blocks are self-describing and may repeat in any order; sampling may also end a block with the standalone <|content_model_end_sampling|> token. The tool-call payload is a single JSON object whose name is extracted by the engine's name-from-args path and whose args object is carved out of the wrapper by _inkling_arg_converter.

Note the terminal labels: THINK_START/THINK_END are what the engine keys its reasoning plumbing on (is_reasoning_end, count_reasoning_tokens, initial-state seeding), so <|end_message|> is labelled THINK_END here even though it ends every block kind — the transition table, not the label, carries the semantics.

Classes:

InklingParser

Bases: ParserEngine

Methods:

Source code in vllm/parser/inkling.py
class InklingParser(ParserEngine):
    CONFIG_NAME = "inkling"

    def __init__(
        self,
        tokenizer: TokenizerLike,
        tools: list[Tool] | None = None,
        **kwargs,
    ) -> None:
        kwargs.setdefault("parser_engine_config", inkling_config())
        super().__init__(tokenizer, tools, **kwargs)
        # Inkling2 spells <|content_xml|> natively and has no audio tokens.
        # Its prompts end between messages: the model opens its own turn.
        vocab = self.vocab
        self._is_inkling2 = CONTENT_XML in vocab and CONTENT_AUDIO_INPUT not in vocab

    def adjust_request(
        self, request: ChatCompletionRequest | ResponsesRequest
    ) -> ChatCompletionRequest | ResponsesRequest:
        """Frame an Inkling2 JSON response format with the block grammar.

        As in the reference, any number of free thinking blocks precede the
        final text block, whose payload alone must match the JSON constraint;
        the API then returns that payload as the content.
        """
        request = super().adjust_request(request)
        if not self._is_inkling2:
            return request
        structured_outputs = request.extract_structured_outputs()
        if structured_outputs is None or (
            structured_outputs.json is None and not structured_outputs.json_object
        ):
            return request
        if request.tools and request.tool_choice not in ("auto", "none", None):
            # TODO: constrain required and named tool choices to tool calls.
            return request
        json_format = structured_outputs_to_format(structured_outputs)

        vocab = self.vocab
        model, text, thinking, end = (
            TokenFormat(token=vocab[token])
            for token in (MESSAGE_MODEL, CONTENT_TEXT, CONTENT_THINKING, END_MESSAGE)
        )
        prefix = None
        if isinstance(request, ChatCompletionRequest) and (
            request.continue_final_message and request.messages
        ):
            prefix = continuation_prefix(request.messages[-1])
        if prefix:
            raise ValueError(
                "Inkling JSON response formats cannot continue a nonempty "
                "assistant message."
            )
        output_format: Format
        if prefix is None:
            excludes = [token for token in INKLING_CONTROL_TOKENS if token in vocab]
            thinking_block = SequenceFormat(
                elements=[model, thinking, AnyTextFormat(excludes=excludes), end]
            )
            output_format = SequenceFormat(
                elements=[
                    StarFormat(content=thinking_block),
                    model,
                    text,
                    json_format,
                    end,
                ]
            )
        else:
            output_format = SequenceFormat(elements=[json_format, end])
        tag = StructuralTag(format=output_format)
        grammar = StructuredOutputsParams(structural_tag=json.dumps(tag.model_dump()))
        if isinstance(request, ChatCompletionRequest):
            request.structured_outputs = grammar
            request.response_format = None
        else:
            request._parser_structured_outputs = grammar
        request._grammar_from_parser = True
        return request

    def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
        """Seed the initial parsing state from the prompt tail.

        Mirrors the Rust parser's `initialize()`: scanning the prompt
        backwards, the last relevant special token decides whether generation
        continues inside a thinking block, a text block, a model message
        header, or between blocks.
        """
        vocab = self.vocab
        thinking_id = vocab.get(CONTENT_THINKING)
        text_id = vocab.get(CONTENT_TEXT)
        model_id = vocab.get(MESSAGE_MODEL)
        special_ids = {vocab[text] for text in INKLING_SPECIAL_TOKENS if text in vocab}
        for token_id in reversed(prompt_token_ids):
            if token_id == thinking_id:
                self._engine.reset(initial_state=ParserState.REASONING)
                self._streaming_initialized = True
                return
            if token_id == text_id:
                self._engine.reset(initial_state=ParserState.CONTENT)
                self._streaming_initialized = True
                return
            if token_id == model_id:
                self._engine.reset(initial_state=ParserState.MESSAGE_HEADER)
                self._streaming_initialized = True
                return
            if token_id in special_ids:
                break

    def is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
        if self._is_inkling2:
            return self._inkling2_is_reasoning_end(input_ids)
        vocab = self.vocab
        reasoning_ids = {vocab.get(CONTENT_THINKING), vocab.get(MESSAGE_MODEL)}
        ended_ids = {vocab.get(CONTENT_TEXT), vocab.get(CONTENT_MODEL_END_SAMPLING)}
        i = rfind_tokens(input_ids, reasoning_ids | ended_ids)
        return i >= 0 and input_ids[i] in ended_ids

    def _inkling2_is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
        """Whether the latest block is a text block of a model message.

        A prompt that ends between messages has not ended reasoning: the
        model may open a thinking block.
        """
        vocab = self.vocab
        text_id = vocab[CONTENT_TEXT]
        block_ids = {
            text_id,
            vocab[CONTENT_THINKING],
            vocab[CONTENT_MODEL_END_SAMPLING],
        }
        role_ids = {
            vocab[token]
            for token in (MESSAGE_MODEL, MESSAGE_USER, MESSAGE_SYSTEM, MESSAGE_TOOL)
        }
        latest_block = None
        for token_id in reversed(input_ids):
            if token_id in role_ids:
                return latest_block == text_id and token_id == vocab[MESSAGE_MODEL]
            if latest_block is None and token_id in block_ids:
                latest_block = token_id
        return latest_block == text_id

    def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
        vocab = self.vocab
        thinking_id = vocab.get(CONTENT_THINKING)
        end_ids = {
            token_id
            for token_id in (
                vocab.get(END_MESSAGE),
                vocab.get(CONTENT_MODEL_END_SAMPLING),
            )
            if token_id is not None
        }
        in_reasoning = False
        count = 0
        for token_id in token_ids:
            if token_id == thinking_id:
                in_reasoning = True
                continue
            if token_id in end_ids:
                in_reasoning = False
                continue
            if in_reasoning:
                count += 1
        return count

    def _single_pass_parse(
        self,
        text: str,
        token_ids: Sequence[int],
        initial_state: ParserState | None = None,
    ) -> tuple[str | None, str | None, ExtractedToolCallInformation]:
        reasoning, content, tool_call_info = super()._single_pass_parse(
            text, token_ids, initial_state=initial_state
        )
        # The engine defers content that follows tool-call events within a
        # single pass; Inkling allows text blocks after tool-call blocks, so
        # flush the trailing text (matching the Rust unified parser).
        if self._deferred_content:
            trailing = self._deferred_content
            self._deferred_content = ""
            content = self._strip_content_whitespace(
                (content or "") + trailing,
                tool_call_info.tools_called,
            )
            tool_call_info = ExtractedToolCallInformation(
                tools_called=tool_call_info.tools_called,
                tool_calls=tool_call_info.tool_calls,
                content=content,
            )
        return reasoning, content, tool_call_info

    @staticmethod
    def _extract_args_value(parsed: dict) -> str | None:
        # Inkling wraps arguments under "args" rather than "arguments".
        for key in ("args", "arguments", "parameters"):
            if key in parsed:
                val = parsed[key]
                if isinstance(val, str):
                    return val
                return json.dumps(val, ensure_ascii=False)
        return None

_inkling2_is_reasoning_end(input_ids)

Whether the latest block is a text block of a model message.

A prompt that ends between messages has not ended reasoning: the model may open a thinking block.

Source code in vllm/parser/inkling.py
def _inkling2_is_reasoning_end(self, input_ids: Sequence[int]) -> bool:
    """Whether the latest block is a text block of a model message.

    A prompt that ends between messages has not ended reasoning: the
    model may open a thinking block.
    """
    vocab = self.vocab
    text_id = vocab[CONTENT_TEXT]
    block_ids = {
        text_id,
        vocab[CONTENT_THINKING],
        vocab[CONTENT_MODEL_END_SAMPLING],
    }
    role_ids = {
        vocab[token]
        for token in (MESSAGE_MODEL, MESSAGE_USER, MESSAGE_SYSTEM, MESSAGE_TOOL)
    }
    latest_block = None
    for token_id in reversed(input_ids):
        if token_id in role_ids:
            return latest_block == text_id and token_id == vocab[MESSAGE_MODEL]
        if latest_block is None and token_id in block_ids:
            latest_block = token_id
    return latest_block == text_id

adjust_initial_state_from_prompt(prompt_token_ids)

Seed the initial parsing state from the prompt tail.

Mirrors the Rust parser's initialize(): scanning the prompt backwards, the last relevant special token decides whether generation continues inside a thinking block, a text block, a model message header, or between blocks.

Source code in vllm/parser/inkling.py
def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
    """Seed the initial parsing state from the prompt tail.

    Mirrors the Rust parser's `initialize()`: scanning the prompt
    backwards, the last relevant special token decides whether generation
    continues inside a thinking block, a text block, a model message
    header, or between blocks.
    """
    vocab = self.vocab
    thinking_id = vocab.get(CONTENT_THINKING)
    text_id = vocab.get(CONTENT_TEXT)
    model_id = vocab.get(MESSAGE_MODEL)
    special_ids = {vocab[text] for text in INKLING_SPECIAL_TOKENS if text in vocab}
    for token_id in reversed(prompt_token_ids):
        if token_id == thinking_id:
            self._engine.reset(initial_state=ParserState.REASONING)
            self._streaming_initialized = True
            return
        if token_id == text_id:
            self._engine.reset(initial_state=ParserState.CONTENT)
            self._streaming_initialized = True
            return
        if token_id == model_id:
            self._engine.reset(initial_state=ParserState.MESSAGE_HEADER)
            self._streaming_initialized = True
            return
        if token_id in special_ids:
            break

adjust_request(request)

Frame an Inkling2 JSON response format with the block grammar.

As in the reference, any number of free thinking blocks precede the final text block, whose payload alone must match the JSON constraint; the API then returns that payload as the content.

Source code in vllm/parser/inkling.py
def adjust_request(
    self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
    """Frame an Inkling2 JSON response format with the block grammar.

    As in the reference, any number of free thinking blocks precede the
    final text block, whose payload alone must match the JSON constraint;
    the API then returns that payload as the content.
    """
    request = super().adjust_request(request)
    if not self._is_inkling2:
        return request
    structured_outputs = request.extract_structured_outputs()
    if structured_outputs is None or (
        structured_outputs.json is None and not structured_outputs.json_object
    ):
        return request
    if request.tools and request.tool_choice not in ("auto", "none", None):
        # TODO: constrain required and named tool choices to tool calls.
        return request
    json_format = structured_outputs_to_format(structured_outputs)

    vocab = self.vocab
    model, text, thinking, end = (
        TokenFormat(token=vocab[token])
        for token in (MESSAGE_MODEL, CONTENT_TEXT, CONTENT_THINKING, END_MESSAGE)
    )
    prefix = None
    if isinstance(request, ChatCompletionRequest) and (
        request.continue_final_message and request.messages
    ):
        prefix = continuation_prefix(request.messages[-1])
    if prefix:
        raise ValueError(
            "Inkling JSON response formats cannot continue a nonempty "
            "assistant message."
        )
    output_format: Format
    if prefix is None:
        excludes = [token for token in INKLING_CONTROL_TOKENS if token in vocab]
        thinking_block = SequenceFormat(
            elements=[model, thinking, AnyTextFormat(excludes=excludes), end]
        )
        output_format = SequenceFormat(
            elements=[
                StarFormat(content=thinking_block),
                model,
                text,
                json_format,
                end,
            ]
        )
    else:
        output_format = SequenceFormat(elements=[json_format, end])
    tag = StructuralTag(format=output_format)
    grammar = StructuredOutputsParams(structural_tag=json.dumps(tag.model_dump()))
    if isinstance(request, ChatCompletionRequest):
        request.structured_outputs = grammar
        request.response_format = None
    else:
        request._parser_structured_outputs = grammar
    request._grammar_from_parser = True
    return request

_inkling_arg_converter(raw_args, partial)

Carve the args object out of the tool-call JSON wrapper.

Why a converter at all: the engine's tool_args_json machinery treats the entire TOOL_ARGS text as the tool arguments, but Inkling's payload is the {"name":...,"args":{...}} wrapper — without a converter, _compute_arg_delta streams the wrapper verbatim into the OpenAI arguments field (converter is None -> raw delta, unconditionally; stream_arg_deltas=False does not stop it). extract_json_member_object returns a prefix-stable verbatim substring, which the engine's argument-delta diffing requires.

Source code in vllm/parser/inkling.py
def _inkling_arg_converter(raw_args: str, partial: bool) -> str:
    """Carve the `args` object out of the tool-call JSON wrapper.

    Why a converter at all: the engine's `tool_args_json` machinery
    treats the *entire* TOOL_ARGS text as the tool arguments, but Inkling's
    payload is the `{"name":...,"args":{...}}` wrapper — without a
    converter, `_compute_arg_delta` streams the wrapper verbatim into
    the OpenAI `arguments` field (`converter is None -> raw delta`,
    unconditionally; `stream_arg_deltas=False` does not stop it).
    `extract_json_member_object` returns a prefix-stable verbatim
    substring, which the engine's argument-delta diffing requires.
    """
    span = extract_json_member_object(raw_args, "args")
    if span is None:
        # No args value yet (streaming) or none at all (treat as empty).
        return "" if partial else "{}"
    return span