vllm_mlx.models.llm

MLX Language Model wrapper.

View the complete module source at #L1-L422.

API details

Each callable below includes its exact signature, type annotations, inputs, defaults, return contract, documented exceptions, implementation source, and parsed docstring sections when the source provides them.

vllm_mlx.models.llm

MLX Language Model wrapper.

This module provides a wrapper around mlx-lm for LLM inference, integrating with vLLM's model execution system.

vllm_mlx.models.llm.logger module-attribute

logger = logging.getLogger(__name__)

vllm_mlx.models.llm.GenerationOutput dataclass

GenerationOutput(text: str, tokens: list[int], finish_reason: str | None = None)

Output from text generation.

vllm_mlx.models.llm.GenerationOutput.text instance-attribute

text: str

vllm_mlx.models.llm.GenerationOutput.tokens instance-attribute

tokens: list[int]

vllm_mlx.models.llm.GenerationOutput.finish_reason class-attribute instance-attribute

finish_reason: str | None = None

vllm_mlx.models.llm.StreamingOutput dataclass

StreamingOutput(text: str, token: int, finished: bool = False, finish_reason: str | None = None, prompt_tokens: int = 0)

Streaming output chunk.

vllm_mlx.models.llm.StreamingOutput.text instance-attribute

text: str

vllm_mlx.models.llm.StreamingOutput.token instance-attribute

token: int

vllm_mlx.models.llm.StreamingOutput.finished class-attribute instance-attribute

finished: bool = False

vllm_mlx.models.llm.StreamingOutput.finish_reason class-attribute instance-attribute

finish_reason: str | None = None

vllm_mlx.models.llm.StreamingOutput.prompt_tokens class-attribute instance-attribute

prompt_tokens: int = 0

vllm_mlx.models.llm.MLXLanguageModel

MLXLanguageModel(model_name: str, tokenizer_name: str | None = None, trust_remote_code: bool = False, mtp: bool = False, mtp_num_draft_tokens: int = 1)

Wrapper around mlx-lm for LLM inference.

This class provides a unified interface for loading and running inference on language models using Apple's MLX framework.

Example

model = MLXLanguageModel("mlx-community/Llama-3.2-3B-Instruct-4bit") output = model.generate("Hello, how are you?", max_tokens=100) print(output.text)

Initialize the MLX language model.

Parameters:

• model_name (str) –

  HuggingFace model name or local path

• tokenizer_name (str | None, default: None ) –

  Optional separate tokenizer name

• trust_remote_code (bool, default: False ) –

  Whether to trust remote code

• mtp (bool, default: False ) –

  Enable native MTP speculative decoding (model must have MTP head)

• mtp_num_draft_tokens (int, default: 1 ) –

  Draft tokens per speculative MTP step

Source code in vllm_mlx/models/llm.py

def __init__(
    self,
    model_name: str,
    tokenizer_name: str | None = None,
    trust_remote_code: bool = False,
    mtp: bool = False,
    mtp_num_draft_tokens: int = 1,
):
    """
    Initialize the MLX language model.

    Args:
        model_name: HuggingFace model name or local path
        tokenizer_name: Optional separate tokenizer name
        trust_remote_code: Whether to trust remote code
        mtp: Enable native MTP speculative decoding (model must have MTP head)
        mtp_num_draft_tokens: Draft tokens per speculative MTP step
    """
    self.model_name = model_name
    self.tokenizer_name = tokenizer_name or model_name
    self.trust_remote_code = trust_remote_code
    self._mtp = mtp
    self._mtp_num_draft_tokens = mtp_num_draft_tokens

    self.model = None
    self.tokenizer = None
    self._loaded = False

vllm_mlx.models.llm.MLXLanguageModel.model_name instance-attribute

model_name = model_name

vllm_mlx.models.llm.MLXLanguageModel.tokenizer_name instance-attribute

tokenizer_name = tokenizer_name or model_name

vllm_mlx.models.llm.MLXLanguageModel.trust_remote_code instance-attribute

trust_remote_code = trust_remote_code

vllm_mlx.models.llm.MLXLanguageModel._mtp instance-attribute

_mtp = mtp

vllm_mlx.models.llm.MLXLanguageModel._mtp_num_draft_tokens instance-attribute

_mtp_num_draft_tokens = mtp_num_draft_tokens

vllm_mlx.models.llm.MLXLanguageModel.model instance-attribute

model = None

vllm_mlx.models.llm.MLXLanguageModel.tokenizer instance-attribute

tokenizer = None

vllm_mlx.models.llm.MLXLanguageModel._loaded instance-attribute

_loaded = False

vllm_mlx.models.llm.MLXLanguageModel.load

load() -> None

Load the model and tokenizer.

Source code in vllm_mlx/models/llm.py

def load(self) -> None:
    """Load the model and tokenizer."""
    if self._loaded:
        return

    try:
        from ..utils.tokenizer import load_model_with_fallback

        logger.info(f"Loading model: {self.model_name}")

        # Build tokenizer config
        tokenizer_config = {"trust_remote_code": self.trust_remote_code}

        # Qwen3 fix: eos_token changed from <|im_end|> to <|endoftext|>
        # but chat template still uses <|im_end|>, so we need to set it explicitly
        if "qwen3" in self.model_name.lower() or "Qwen3" in self.model_name:
            tokenizer_config["eos_token"] = "<|im_end|>"
            logger.info("Qwen3 detected: setting eos_token to <|im_end|>")

        self.model, self.tokenizer = load_model_with_fallback(
            self.model_name,
            tokenizer_config=tokenizer_config,
        )

        self._loaded = True
        logger.info(f"Model loaded successfully: {self.model_name}")

    except ImportError as err:
        raise ImportError(
            "mlx-lm is required for LLM inference. Install with: pip install mlx-lm"
        ) from err
    except Exception as e:
        logger.error(f"Failed to load model: {e}")
        raise

vllm_mlx.models.llm.MLXLanguageModel._create_sampler

_create_sampler(temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0)

Create a sampler for text generation.

Source code in vllm_mlx/models/llm.py

def _create_sampler(
    self,
    temperature: float = 0.7,
    top_p: float = 0.9,
    top_k: int = 0,
    min_p: float = 0.0,
):
    """Create a sampler for text generation."""
    from mlx_lm.sample_utils import make_sampler

    return make_sampler(
        temp=temperature,
        top_p=top_p,
        top_k=top_k,
        min_p=min_p,
    )

vllm_mlx.models.llm.MLXLanguageModel._create_logits_processors

_create_logits_processors(presence_penalty: float = 0.0, repetition_penalty: float = 1.0)

Create logits processors for penalty-based sampling.

Source code in vllm_mlx/models/llm.py

def _create_logits_processors(
    self,
    presence_penalty: float = 0.0,
    repetition_penalty: float = 1.0,
):
    """Create logits processors for penalty-based sampling."""
    from mlx_lm.sample_utils import make_logits_processors

    processors = make_logits_processors(
        repetition_penalty=(
            repetition_penalty if repetition_penalty != 1.0 else None
        ),
        presence_penalty=presence_penalty if presence_penalty != 0.0 else None,
    )
    return processors if processors else None

vllm_mlx.models.llm.MLXLanguageModel.generate

generate(prompt: str, max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0, presence_penalty: float = 0.0, repetition_penalty: float = 1.0, stop: list[str] | None = None, logits_processors: list | None = None, **kwargs) -> GenerationOutput

Generate text from a prompt.

Parameters:

• prompt (str) –

  Input prompt text

• max_tokens (int, default: 256 ) –

  Maximum number of tokens to generate

• temperature (float, default: 0.7 ) –

  Sampling temperature (0 = greedy)

• top_p (float, default: 0.9 ) –

  Top-p (nucleus) sampling parameter

• top_k (int, default: 0 ) –

  Top-k sampling (0 = disabled)

• min_p (float, default: 0.0 ) –

  Minimum probability threshold

• presence_penalty (float, default: 0.0 ) –

  Additive penalty for token presence

• repetition_penalty (float, default: 1.0 ) –

  Multiplicative penalty for repeating tokens

• stop (list[str] | None, default: None ) –

  List of stop sequences

• logits_processors (list | None, default: None ) –

  Optional externally-supplied logits processors (e.g. JSON schema constrained decoding). Merged with built-in penalty processors.

Returns:

• GenerationOutput –

  GenerationOutput with generated text and tokens

Source code in vllm_mlx/models/llm.py

def generate(
    self,
    prompt: str,
    max_tokens: int = 256,
    temperature: float = 0.7,
    top_p: float = 0.9,
    top_k: int = 0,
    min_p: float = 0.0,
    presence_penalty: float = 0.0,
    repetition_penalty: float = 1.0,
    stop: list[str] | None = None,
    logits_processors: list | None = None,
    **kwargs,
) -> GenerationOutput:
    """
    Generate text from a prompt.

    Args:
        prompt: Input prompt text
        max_tokens: Maximum number of tokens to generate
        temperature: Sampling temperature (0 = greedy)
        top_p: Top-p (nucleus) sampling parameter
        top_k: Top-k sampling (0 = disabled)
        min_p: Minimum probability threshold
        presence_penalty: Additive penalty for token presence
        repetition_penalty: Multiplicative penalty for repeating tokens
        stop: List of stop sequences
        logits_processors: Optional externally-supplied logits processors
            (e.g. JSON schema constrained decoding).  Merged with built-in
            penalty processors.

    Returns:
        GenerationOutput with generated text and tokens
    """
    if not self._loaded:
        self.load()

    from mlx_lm import generate

    # Create sampler and logits processors with full Unsloth params
    sampler = self._create_sampler(temperature, top_p, top_k, min_p)
    penalty_processors = self._create_logits_processors(
        presence_penalty, repetition_penalty
    )
    # Merge any externally-provided logits_processors with penalty processors
    all_processors = penalty_processors or []
    if logits_processors:
        all_processors = list(logits_processors) + all_processors

    # Generate text
    output_text = generate(
        self.model,
        self.tokenizer,
        prompt=prompt,
        max_tokens=max_tokens,
        sampler=sampler,
        logits_processors=all_processors if all_processors else None,
        verbose=False,
    )

    # Tokenize output to get token IDs
    tokens = self.tokenizer.encode(output_text)

    # Determine finish reason
    finish_reason = "length" if len(tokens) >= max_tokens else "stop"

    return GenerationOutput(
        text=output_text,
        tokens=tokens,
        finish_reason=finish_reason,
    )

vllm_mlx.models.llm.MLXLanguageModel.stream_generate

stream_generate(prompt: Union[str, array, list[int]], max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0, presence_penalty: float = 0.0, repetition_penalty: float = 1.0, stop: list[str] | None = None, logits_processors: list | None = None, prompt_cache=None, **kwargs) -> Iterator[StreamingOutput]

Stream text generation token by token.

Parameters:

• prompt (Union[str, array, list[int]]) –

  Input prompt text, token array, or token id list

• max_tokens (int, default: 256 ) –

  Maximum number of tokens to generate

• temperature (float, default: 0.7 ) –

  Sampling temperature (0 = greedy)

• top_p (float, default: 0.9 ) –

  Top-p (nucleus) sampling parameter

• top_k (int, default: 0 ) –

  Top-k sampling (0 = disabled)

• min_p (float, default: 0.0 ) –

  Minimum probability threshold

• presence_penalty (float, default: 0.0 ) –

  Additive penalty for token presence

• repetition_penalty (float, default: 1.0 ) –

  Multiplicative penalty for repeating tokens

• stop (list[str] | None, default: None ) –

  List of stop sequences

• prompt_cache –

  Pre-populated KV cache (e.g. from SpecPrefill)

Yields:

• StreamingOutput –

  StreamingOutput for each generated token

Source code in vllm_mlx/models/llm.py

def stream_generate(
    self,
    prompt: Union[str, "mx.array", list[int]],
    max_tokens: int = 256,
    temperature: float = 0.7,
    top_p: float = 0.9,
    top_k: int = 0,
    min_p: float = 0.0,
    presence_penalty: float = 0.0,
    repetition_penalty: float = 1.0,
    stop: list[str] | None = None,
    logits_processors: list | None = None,
    prompt_cache=None,
    **kwargs,
) -> Iterator[StreamingOutput]:
    """
    Stream text generation token by token.

    Args:
        prompt: Input prompt text, token array, or token id list
        max_tokens: Maximum number of tokens to generate
        temperature: Sampling temperature (0 = greedy)
        top_p: Top-p (nucleus) sampling parameter
        top_k: Top-k sampling (0 = disabled)
        min_p: Minimum probability threshold
        presence_penalty: Additive penalty for token presence
        repetition_penalty: Multiplicative penalty for repeating tokens
        stop: List of stop sequences
        prompt_cache: Pre-populated KV cache (e.g. from SpecPrefill)

    Yields:
        StreamingOutput for each generated token
    """
    if not self._loaded:
        self.load()

    from mlx_lm import stream_generate

    # Create sampler and logits processors with full Unsloth params
    sampler = self._create_sampler(temperature, top_p, top_k, min_p)
    penalty_processors = self._create_logits_processors(
        presence_penalty, repetition_penalty
    )
    # Merge any externally-provided logits_processors with penalty processors
    all_processors = None
    if penalty_processors or logits_processors:
        all_processors = (logits_processors or []) + (penalty_processors or [])

    # Count prompt tokens once upfront
    if isinstance(prompt, str):
        num_prompt_tokens = len(self.tokenizer.encode(prompt))
    else:
        num_prompt_tokens = len(prompt)

    stop_list = stop or []
    max_stop_len = max((len(s) for s in stop_list), default=0)
    accumulated_tail = ""

    mtp_kwargs = {}
    if self._mtp:
        mtp_kwargs["num_draft_tokens"] = self._mtp_num_draft_tokens
    if prompt_cache is not None:
        mtp_kwargs["prompt_cache"] = prompt_cache

    for token_count, response in enumerate(
        stream_generate(
            self.model,
            self.tokenizer,
            prompt=prompt,
            max_tokens=max_tokens,
            sampler=sampler,
            logits_processors=all_processors,
            **mtp_kwargs,
        ),
        start=1,
    ):
        # response.text is the new token text (not accumulated)
        new_text = response.text

        # Check for stop sequences against a bounded tail rather than
        # accumulating the full response (which is O(n^2) over the loop).
        should_stop = False
        if max_stop_len > 0:
            combined = accumulated_tail + new_text
            for stop_seq in stop_list:
                if stop_seq in combined:
                    should_stop = True
                    break
            accumulated_tail = combined[-max_stop_len:]

        finished = should_stop or token_count >= max_tokens
        finish_reason = None
        if finished:
            finish_reason = "stop" if should_stop else "length"

        yield StreamingOutput(
            text=new_text,
            token=response.token if hasattr(response, "token") else 0,
            finished=finished,
            finish_reason=finish_reason,
            prompt_tokens=num_prompt_tokens,
        )

        if finished:
            break

vllm_mlx.models.llm.MLXLanguageModel.chat

chat(messages: list[dict], max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, tools: list | None = None, chat_template_kwargs: dict | None = None, **kwargs) -> GenerationOutput

Generate a chat response.

Parameters:

• messages (list[dict]) –

  List of chat messages [{"role": "user", "content": "..."}]

• max_tokens (int, default: 256 ) –

  Maximum tokens to generate

• temperature (float, default: 0.7 ) –

  Sampling temperature

• top_p (float, default: 0.9 ) –

  Top-p sampling parameter

• tools (list | None, default: None ) –

  Optional list of tools for function calling

• **kwargs –

  Additional generation parameters

Returns:

• GenerationOutput –

  GenerationOutput with the assistant's response

Source code in vllm_mlx/models/llm.py

def chat(
    self,
    messages: list[dict],
    max_tokens: int = 256,
    temperature: float = 0.7,
    top_p: float = 0.9,
    tools: list | None = None,
    chat_template_kwargs: dict | None = None,
    **kwargs,
) -> GenerationOutput:
    """
    Generate a chat response.

    Args:
        messages: List of chat messages [{"role": "user", "content": "..."}]
        max_tokens: Maximum tokens to generate
        temperature: Sampling temperature
        top_p: Top-p sampling parameter
        tools: Optional list of tools for function calling
        **kwargs: Additional generation parameters

    Returns:
        GenerationOutput with the assistant's response
    """
    if not self._loaded:
        self.load()

    # Apply chat template
    if hasattr(self.tokenizer, "apply_chat_template"):
        # Build kwargs for apply_chat_template
        template_kwargs = {
            "tokenize": False,
            "add_generation_prompt": True,
        }

        # Add tools if provided and supported
        if tools:
            template_kwargs["tools"] = tools
        if chat_template_kwargs:
            template_kwargs.update(chat_template_kwargs)

        try:
            prompt = self.tokenizer.apply_chat_template(
                messages,
                **template_kwargs,
            )
        except TypeError:
            # Tokenizer doesn't support all requested template kwargs
            template_kwargs.pop("tools", None)
            for key in (chat_template_kwargs or {}).keys():
                template_kwargs.pop(key, None)
            prompt = self.tokenizer.apply_chat_template(
                messages,
                **template_kwargs,
            )
    else:
        # Fallback: simple concatenation
        prompt = "\n".join(f"{msg['role']}: {msg['content']}" for msg in messages)
        prompt += "\nassistant:"

    return self.generate(
        prompt=prompt,
        max_tokens=max_tokens,
        temperature=temperature,
        top_p=top_p,
        **kwargs,
    )

vllm_mlx.models.llm.MLXLanguageModel.get_model_info

get_model_info() -> dict

Get information about the loaded model.

Source code in vllm_mlx/models/llm.py

def get_model_info(self) -> dict:
    """Get information about the loaded model."""
    if not self._loaded:
        return {"loaded": False, "model_name": self.model_name}

    info = {
        "loaded": True,
        "model_name": self.model_name,
        "tokenizer_name": self.tokenizer_name,
    }

    # Try to get model config
    if hasattr(self.model, "config"):
        config = self.model.config
        info.update(
            {
                "vocab_size": getattr(config, "vocab_size", None),
                "hidden_size": getattr(config, "hidden_size", None),
                "num_layers": getattr(config, "num_hidden_layers", None),
                "num_heads": getattr(config, "num_attention_heads", None),
            }
        )

    return info

vllm_mlx.models.llm.MLXLanguageModel.__repr__

__repr__() -> str

Source code in vllm_mlx/models/llm.py

def __repr__(self) -> str:
    status = "loaded" if self._loaded else "not loaded"
    return f"<MLXLanguageModel model={self.model_name} status={status}>"

Complete contract reference

Expand any definition for its exact inputs, annotations, defaults, return contract, directly raised exceptions, source-grounded behavior, and immutable line link. This section includes private and nested definitions that ordinary API generators omit.

vllm_mlx.models.llm.GenerationOutput · class

vllm_mlx.models.llm.GenerationOutput(text: str, tokens: list[int], finish_reason: str | None = None)

Output from text generation.

Parameters

Name	Type	Required	Default	Description
text	str	yes	none	Required constructor field.
tokens	list[int]	yes	none	Required constructor field.
finish_reason	str \| None	no	None	Optional constructor field; defaults to None.

Returns

• Constructs: vllm_mlx.models.llm.GenerationOutput

Exceptions and behavior

Class GenerationOutput declares 0 direct member(s). No direct raise statement appears in this definition.

View source #L21-L26.

vllm_mlx.models.llm.StreamingOutput · class

vllm_mlx.models.llm.StreamingOutput(text: str, token: int, finished: bool = False, finish_reason: str | None = None, prompt_tokens: int = 0)

Streaming output chunk.

Parameters

Name	Type	Required	Default	Description
text	str	yes	none	Required constructor field.
token	int	yes	none	Required constructor field.
finished	bool	no	False	Optional constructor field; defaults to False.
finish_reason	str \| None	no	None	Optional constructor field; defaults to None.
prompt_tokens	int	no	0	Optional constructor field; defaults to 0.

Returns

• Constructs: vllm_mlx.models.llm.StreamingOutput

Exceptions and behavior

Class StreamingOutput declares 0 direct member(s). No direct raise statement appears in this definition.

View source #L30-L37.

vllm_mlx.models.llm.MLXLanguageModel · class

vllm_mlx.models.llm.MLXLanguageModel(model_name: str, tokenizer_name: str | None = None, trust_remote_code: bool = False, mtp: bool = False, mtp_num_draft_tokens: int = 1)

Wrapper around mlx-lm for LLM inference.

Parameters

Name	Type	Required	Default	Description
model_name	str	yes	none	HuggingFace model name or local path
tokenizer_name	str \| None	no	None	Optional separate tokenizer name
trust_remote_code	bool	no	False	Whether to trust remote code
mtp	bool	no	False	Enable native MTP speculative decoding (model must have MTP head)
mtp_num_draft_tokens	int	no	1	Draft tokens per speculative MTP step

Returns

• Constructs: vllm_mlx.models.llm.MLXLanguageModel

Exceptions and behavior

Class MLXLanguageModel declares 9 direct member(s). No direct raise statement appears in this definition.

View source #L40-L422.

vllm_mlx.models.llm.MLXLanguageModel.__init__ · method

vllm_mlx.models.llm.MLXLanguageModel.__init__(model_name: str, tokenizer_name: str | None = None, trust_remote_code: bool = False, mtp: bool = False, mtp_num_draft_tokens: int = 1) -> not annotated

Initialize the MLX language model.

Parameters

Name	Type	Required	Default	Description
model_name	str	yes	none	HuggingFace model name or local path
tokenizer_name	str \| None	no	None	Optional separate tokenizer name
trust_remote_code	bool	no	False	Whether to trust remote code
mtp	bool	no	False	Enable native MTP speculative decoding (model must have MTP head)
mtp_num_draft_tokens	int	no	1	Draft tokens per speculative MTP step

Returns

• Type: not annotated

Exceptions and behavior

Method MLXLanguageModel.__init__ updates self.model_name, self.tokenizer_name, self.trust_remote_code, self._mtp. No direct raise statement appears in this definition.

View source #L53-L79.

vllm_mlx.models.llm.MLXLanguageModel.load · method

vllm_mlx.models.llm.MLXLanguageModel.load() -> None

Load the model and tokenizer.

Parameters

This callable has no explicit inputs.

Returns

• Type: None
• Direct return expressions: None

Exceptions and behavior

Method MLXLanguageModel.load updates self.model, self.tokenizer, self._loaded; calls logger.info, self.model_name.lower, load_model_with_fallback, ImportError; can raise ImportError; returns None. Directly raised exceptions: ImportError.

View source #L81-L114.

vllm_mlx.models.llm.MLXLanguageModel._create_sampler · method

vllm_mlx.models.llm.MLXLanguageModel._create_sampler(temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0) -> not annotated

Create a sampler for text generation.

Parameters

Name	Type	Required	Default	Description
temperature	float	no	0.7	Optional positional or keyword input; defaults to 0.7.
top_p	float	no	0.9	Optional positional or keyword input; defaults to 0.9.
top_k	int	no	0	Optional positional or keyword input; defaults to 0.
min_p	float	no	0.0	Optional positional or keyword input; defaults to 0.0.

Returns

• Type: not annotated
• Direct return expressions: make_sampler(temp=temperature, top_p=top_p, top_k=top_k, min_p=min_p)

Exceptions and behavior

Method MLXLanguageModel._create_sampler calls make_sampler; returns make_sampler(temp=temperature, top_p=top_p, top_k=top_k, min_p=min_p). No direct raise statement appears in this definition.

View source #L116-L131.

vllm_mlx.models.llm.MLXLanguageModel._create_logits_processors · method

vllm_mlx.models.llm.MLXLanguageModel._create_logits_processors(presence_penalty: float = 0.0, repetition_penalty: float = 1.0) -> not annotated

Create logits processors for penalty-based sampling.

Parameters

Name	Type	Required	Default	Description
presence_penalty	float	no	0.0	Optional positional or keyword input; defaults to 0.0.
repetition_penalty	float	no	1.0	Optional positional or keyword input; defaults to 1.0.

Returns

• Type: not annotated
• Direct return expressions: processors if processors else None

Exceptions and behavior

Method MLXLanguageModel._create_logits_processors calls make_logits_processors; returns processors if processors else None. No direct raise statement appears in this definition.

View source #L133-L147.

vllm_mlx.models.llm.MLXLanguageModel.generate · method

vllm_mlx.models.llm.MLXLanguageModel.generate(prompt: str, max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0, presence_penalty: float = 0.0, repetition_penalty: float = 1.0, stop: list[str] | None = None, logits_processors: list | None = None, **kwargs) -> GenerationOutput

Generate text from a prompt.

Parameters

Name	Type	Required	Default	Description
prompt	str	yes	none	Input prompt text
max_tokens	int	no	256	Maximum number of tokens to generate
temperature	float	no	0.7	Sampling temperature (0 = greedy)
top_p	float	no	0.9	Top-p (nucleus) sampling parameter
top_k	int	no	0	Top-k sampling (0 = disabled)
min_p	float	no	0.0	Minimum probability threshold
presence_penalty	float	no	0.0	Additive penalty for token presence
repetition_penalty	float	no	1.0	Multiplicative penalty for repeating tokens
stop	list[str] \| None	no	None	List of stop sequences
logits_processors	list \| None	no	None	Optional externally-supplied logits processors (e.g. JSON schema constrained decoding). Merged with built-in penalty processors.
**kwargs	not annotated	no	none	Additional variadic keyword inputs accepted by this callable.

Returns

• Type: GenerationOutput
• Direct return expressions: GenerationOutput(text=output_text, tokens=tokens, finish_reason=finish_reason)

Exceptions and behavior

Method MLXLanguageModel.generate calls self.load, self._create_sampler, self._create_logits_processors, list; returns GenerationOutput(text=output_text, tokens=tokens, finish_reason=finish_reason). No direct raise statement appears in this definition.

View source #L149-L219.

vllm_mlx.models.llm.MLXLanguageModel.stream_generate · method

vllm_mlx.models.llm.MLXLanguageModel.stream_generate(prompt: Union[str, 'mx.array', list[int]], max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0, presence_penalty: float = 0.0, repetition_penalty: float = 1.0, stop: list[str] | None = None, logits_processors: list | None = None, prompt_cache = None, **kwargs) -> Iterator[StreamingOutput]

Stream text generation token by token.

Parameters

Name	Type	Required	Default	Description
prompt	Union[str, 'mx.array', list[int]]	yes	none	Input prompt text, token array, or token id list
max_tokens	int	no	256	Maximum number of tokens to generate
temperature	float	no	0.7	Sampling temperature (0 = greedy)
top_p	float	no	0.9	Top-p (nucleus) sampling parameter
top_k	int	no	0	Top-k sampling (0 = disabled)
min_p	float	no	0.0	Minimum probability threshold
presence_penalty	float	no	0.0	Additive penalty for token presence
repetition_penalty	float	no	1.0	Multiplicative penalty for repeating tokens
stop	list[str] \| None	no	None	List of stop sequences
logits_processors	list \| None	no	None	Optional positional or keyword input; defaults to None.
prompt_cache	not annotated	no	None	Pre-populated KV cache (e.g. from SpecPrefill)
**kwargs	not annotated	no	none	Additional variadic keyword inputs accepted by this callable.

Returns

• Type: Iterator[StreamingOutput]
• Yields values incrementally.

Exceptions and behavior

Method MLXLanguageModel.stream_generate calls self.load, self._create_sampler, self._create_logits_processors, isinstance; yields values incrementally. No direct raise statement appears in this definition.

View source #L221-L325.

vllm_mlx.models.llm.MLXLanguageModel.chat · method

vllm_mlx.models.llm.MLXLanguageModel.chat(messages: list[dict], max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, tools: list | None = None, chat_template_kwargs: dict | None = None, **kwargs) -> GenerationOutput

Generate a chat response.

Parameters

Name	Type	Required	Default	Description
messages	list[dict]	yes	none	List of chat messages [{"role": "user", "content": "..."}]
max_tokens	int	no	256	Maximum tokens to generate
temperature	float	no	0.7	Sampling temperature
top_p	float	no	0.9	Top-p sampling parameter
tools	list \| None	no	None	Optional list of tools for function calling
chat_template_kwargs	dict \| None	no	None	Optional positional or keyword input; defaults to None.
**kwargs	not annotated	no	none	Additional generation parameters

Returns

• Type: GenerationOutput
• Direct return expressions: self.generate(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, **kwargs)

Exceptions and behavior

Method MLXLanguageModel.chat calls self.load, hasattr, template_kwargs.update, self.tokenizer.apply_chat_template; returns self.generate(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, **kwargs). No direct raise statement appears in this definition.

View source #L327-L393.

vllm_mlx.models.llm.MLXLanguageModel.get_model_info · method

vllm_mlx.models.llm.MLXLanguageModel.get_model_info() -> dict

Get information about the loaded model.

Parameters

This callable has no explicit inputs.

Returns

• Type: dict
• Direct return expressions: {'loaded': False, 'model_name': self.model_name}; info

Exceptions and behavior

Method MLXLanguageModel.get_model_info calls hasattr, info.update, getattr; has 2 explicit return paths. No direct raise statement appears in this definition.

View source #L395-L418.

vllm_mlx.models.llm.MLXLanguageModel.__repr__ · method

vllm_mlx.models.llm.MLXLanguageModel.__repr__() -> str

Method MLXLanguageModel.__repr__ returns f'<MLXLanguageModel model={self.model_name} status={status}>'.

Parameters

This callable has no explicit inputs.

Returns

• Type: str
• Direct return expressions: f'<MLXLanguageModel model={self.model_name} status={status}>'

Exceptions and behavior

Method MLXLanguageModel.__repr__ returns f'<MLXLanguageModel model={self.model_name} status={status}>'. No direct raise statement appears in this definition.

View source #L420-L422.

Complete symbol map

This map also includes private definitions and nested helpers. The signature column exposes every explicit input even when an internal helper has no dedicated parameter prose.

Symbol	Kind	Signature and inputs	What it does	Source
GenerationOutput	class	GenerationOutput(text: str, tokens: list[int], finish_reason: str \| None = None)	Output from text generation.	#L21-L26
StreamingOutput	class	StreamingOutput(text: str, token: int, finished: bool = False, finish_reason: str \| None = None, prompt_tokens: int = 0)	Streaming output chunk.	#L30-L37
MLXLanguageModel	class	MLXLanguageModel(model_name: str, tokenizer_name: str \| None = None, trust_remote_code: bool = False, mtp: bool = False, mtp_num_draft_tokens: int = 1)	Wrapper around mlx-lm for LLM inference.	#L40-L422
MLXLanguageModel.__init__	method	MLXLanguageModel.__init__(model_name: str, tokenizer_name: str \| None = None, trust_remote_code: bool = False, mtp: bool = False, mtp_num_draft_tokens: int = 1) -> not annotated	Initialize the MLX language model.	#L53-L79
MLXLanguageModel.load	method	MLXLanguageModel.load() -> None	Load the model and tokenizer.	#L81-L114
MLXLanguageModel._create_sampler	method	MLXLanguageModel._create_sampler(temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0) -> not annotated	Create a sampler for text generation.	#L116-L131
MLXLanguageModel._create_logits_processors	method	MLXLanguageModel._create_logits_processors(presence_penalty: float = 0.0, repetition_penalty: float = 1.0) -> not annotated	Create logits processors for penalty-based sampling.	#L133-L147
MLXLanguageModel.generate	method	MLXLanguageModel.generate(prompt: str, max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0, presence_penalty: float = 0.0, repetition_penalty: float = 1.0, stop: list[str] \| None = None, logits_processors: list \| None = None, **kwargs) -> GenerationOutput	Generate text from a prompt.	#L149-L219
MLXLanguageModel.stream_generate	method	MLXLanguageModel.stream_generate(prompt: Union[str, 'mx.array', list[int]], max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, top_k: int = 0, min_p: float = 0.0, presence_penalty: float = 0.0, repetition_penalty: float = 1.0, stop: list[str] \| None = None, logits_processors: list \| None = None, prompt_cache = None, **kwargs) -> Iterator[StreamingOutput]	Stream text generation token by token.	#L221-L325
MLXLanguageModel.chat	method	MLXLanguageModel.chat(messages: list[dict], max_tokens: int = 256, temperature: float = 0.7, top_p: float = 0.9, tools: list \| None = None, chat_template_kwargs: dict \| None = None, **kwargs) -> GenerationOutput	Generate a chat response.	#L327-L393
MLXLanguageModel.get_model_info	method	MLXLanguageModel.get_model_info() -> dict	Get information about the loaded model.	#L395-L418
MLXLanguageModel.__repr__	method	MLXLanguageModel.__repr__() -> str	Method MLXLanguageModel.__repr__ returns f'<MLXLanguageModel model={self.model_name} status={status}>'.	#L420-L422