{ "schema_version": "1.0", "repository": "waybarrios/vllm-mlx", "source_branch": "gh-pages", "source_revision": "a69d47912bcb21d8fe04d48f75fa896b620ffcfa", "module_count": 146, "symbol_count": 2182, "source_roots": [ "vllm_mlx", "scripts", "examples" ], "modules": [ { "name": "examples.audio_separation_example", "path": "examples/audio_separation_example.py", "page_path": "reference/source/examples/audio_separation_example.md", "docstring": "Audio Separation Example - Isolate voice from background using SAM-Audio\n\nSAM-Audio uses text-guided source separation to isolate specific sounds.\n\nUsage:\n python examples/audio_separation_example.py input.mp3\n python examples/audio_separation_example.py input.mp3 --description \"music\"\n python examples/audio_separation_example.py input.mp3 -o voice.wav\n\nModels:\n - mlx-community/sam-audio-large-fp16 (best quality, 3B params)\n - mlx-community/sam-audio-small-fp16 (faster, 0.6B params)", "summary": "Audio Separation Example - Isolate voice from background using SAM-Audio SAM-Audio uses text-guided source separation to isolate specific sounds.", "line_count": 120, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/audio_separation_example.py#L1-L120", "members": [ "main" ], "symbols": [ { "name": "main", "qualname": "main", "full_name": "examples.audio_separation_example.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 25, "end_line": 116, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/audio_separation_example.py#L25-L116", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "os.path.exists", "os.path.splitext", "time.time", "AudioProcessor", "processor.load", "processor.separate", "processor.save", "os.system" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "examples.benchmark_all_models", "path": "examples/benchmark_all_models.py", "page_path": "reference/source/examples/benchmark_all_models.md", "docstring": "Benchmark all text models for README.", "summary": "Benchmark all text models for README.", "line_count": 148, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_all_models.py#L1-L148", "members": [ "benchmark_model", "main" ], "symbols": [ { "name": "benchmark_model", "qualname": "benchmark_model", "full_name": "examples.benchmark_all_models.benchmark_model", "kind": "function", "signature": "def benchmark_model(model_name: str)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Benchmark a single model and return results.", "summary": "Benchmark a single model and return results.", "implementation": "Function `benchmark_model` calls `SamplingParams`, `print`, `load`, `format_prompt`; returns `{'model': model_name.split('/')[-1], 'single_tps': single_tps, 'batch_tps': batch_tps, 'speedup': speedup, 'ttft_ms': t…`.", "documented": true, "public": true, "addressable": true, "line": 7, "end_line": 107, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_all_models.py#L7-L107", "decorators": [], "calls": [ "SamplingParams", "print", "load", "format_prompt", "EngineConfig", "SchedulerConfig", "EngineCore", "time.perf_counter", "engine.generate_batch_sync", "single_times.append", "single_tokens.append", "sum", "engine.scheduler.reset", "model_name.split", "engine.close" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'model': model_name.split('/')[-1], 'single_tps': single_tps, 'batch_tps': batch_tps, 'speedup': speedup, 'ttft_ms': t…" ], "awaits": false, "yields": false }, { "name": "format_prompt", "qualname": "benchmark_model.format_prompt", "full_name": "examples.benchmark_all_models.benchmark_model.format_prompt", "kind": "nested function", "signature": "def format_prompt(p)", "parameters": [ { "name": "p", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `benchmark_model.format_prompt` calls `tokenizer.apply_chat_template`; returns `tokenizer.apply_chat_template([{'role': 'user', 'content': p}], tokenize=False, add_generation_prompt=True)`.", "implementation": "Nested Function `benchmark_model.format_prompt` calls `tokenizer.apply_chat_template`; returns `tokenizer.apply_chat_template([{'role': 'user', 'content': p}], tokenize=False, add_generation_prompt=True)`.", "documented": false, "public": false, "addressable": false, "line": 29, "end_line": 34, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_all_models.py#L29-L34", "decorators": [], "calls": [ "tokenizer.apply_chat_template" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tokenizer.apply_chat_template([{'role': 'user', 'content': p}], tokenize=False, add_generation_prompt=True)" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.benchmark_all_models.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `benchmark_model`, `results.append`, `print`, `traceback.print_exc`.", "implementation": "Function `main` calls `benchmark_model`, `results.append`, `print`, `traceback.print_exc`.", "documented": false, "public": true, "addressable": true, "line": 110, "end_line": 144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_all_models.py#L110-L144", "decorators": [], "calls": [ "benchmark_model", "results.append", "print", "traceback.print_exc" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.benchmark_audio", "path": "examples/benchmark_audio.py", "page_path": "reference/source/examples/benchmark_audio.md", "docstring": "Audio benchmarks for vllm-mlx.\n\nBenchmarks STT (Speech-to-Text), TTS (Text-to-Speech), and audio processing.", "summary": "Audio benchmarks for vllm-mlx.", "line_count": 332, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L1-L332", "members": [ "STT_MODELS", "TTS_MODELS", "TEST_TEXTS", "generate_test_audio", "benchmark_tts", "get_audio_duration", "benchmark_stt", "check_whisper_backend", "run_tts_benchmarks", "run_stt_benchmarks", "main" ], "symbols": [ { "name": "generate_test_audio", "qualname": "generate_test_audio", "full_name": "examples.benchmark_audio.generate_test_audio", "kind": "function", "signature": "def generate_test_audio(duration_seconds: float=5.0) -> str", "parameters": [ { "name": "duration_seconds", "kind": "positional or keyword", "annotation": "float", "default": "5.0", "required": false, "description": "Optional positional or keyword input; defaults to `5.0`." } ], "return_annotation": "str", "docstring": "Generate a simple test audio file using TTS.", "summary": "Generate a simple test audio file using TTS.", "implementation": "Function `generate_test_audio` calls `np.linspace`, `int`, `np.sin`, `(audio * 32767).astype`; returns `path`.", "documented": true, "public": true, "addressable": true, "line": 36, "end_line": 61, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L36-L61", "decorators": [], "calls": [ "np.linspace", "int", "np.sin", "(audio * 32767).astype", "tempfile.mkstemp", "wave.open", "f.setnchannels", "f.setsampwidth", "f.setframerate", "f.writeframes", "audio.tobytes", "os.close" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "path" ], "awaits": false, "yields": false }, { "name": "benchmark_tts", "qualname": "benchmark_tts", "full_name": "examples.benchmark_audio.benchmark_tts", "kind": "function", "signature": "def benchmark_tts(model_name: str, alias: str, texts: list[str], voice: str='af_heart')", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "alias", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "texts", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "voice", "kind": "positional or keyword", "annotation": "str", "default": "'af_heart'", "required": false, "description": "Optional positional or keyword input; defaults to `'af_heart'`." } ], "return_annotation": "not annotated", "docstring": "Benchmark TTS model.", "summary": "Benchmark TTS model.", "implementation": "Function `benchmark_tts` calls `print`, `time.time`, `TTSEngine`, `engine.load`; returns `{'model': alias, 'load_time': load_time, 'avg_chars_per_sec': avg_chars_per_sec, 'avg_rtf': avg_rtf}`.", "documented": true, "public": true, "addressable": true, "line": 64, "end_line": 124, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L64-L124", "decorators": [], "calls": [ "print", "time.time", "TTSEngine", "engine.load", "enumerate", "len", "engine.generate", "results.append", "sum" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'model': alias, 'load_time': load_time, 'avg_chars_per_sec': avg_chars_per_sec, 'avg_rtf': avg_rtf}" ], "awaits": false, "yields": false }, { "name": "get_audio_duration", "qualname": "get_audio_duration", "full_name": "examples.benchmark_audio.get_audio_duration", "kind": "function", "signature": "def get_audio_duration(audio_path: str) -> float", "parameters": [ { "name": "audio_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Get audio duration in seconds.", "summary": "Get audio duration in seconds.", "implementation": "Function `get_audio_duration` calls `audio_path.endswith`, `contextlib.closing`, `wave.open`, `f.getnframes`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 127, "end_line": 159, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L127-L159", "decorators": [], "calls": [ "audio_path.endswith", "contextlib.closing", "wave.open", "f.getnframes", "f.getframerate", "float", "subprocess.run", "result.stdout.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "frames / float(rate)", "float(result.stdout.strip())", "0.0" ], "awaits": false, "yields": false }, { "name": "benchmark_stt", "qualname": "benchmark_stt", "full_name": "examples.benchmark_audio.benchmark_stt", "kind": "function", "signature": "def benchmark_stt(model_name: str, alias: str, audio_path: str)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "alias", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "audio_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Benchmark STT model.", "summary": "Benchmark STT model.", "implementation": "Function `benchmark_stt` calls `print`, `get_audio_duration`, `time.time`, `STTEngine`; returns `{'model': alias, 'load_time': load_time, 'audio_duration': duration, 'trans_time': trans_time, 'rtf': rtf}`.", "documented": true, "public": true, "addressable": true, "line": 162, "end_line": 212, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L162-L212", "decorators": [], "calls": [ "print", "get_audio_duration", "time.time", "STTEngine", "engine.load", "engine.transcribe", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'model': alias, 'load_time': load_time, 'audio_duration': duration, 'trans_time': trans_time, 'rtf': rtf}" ], "awaits": false, "yields": false }, { "name": "check_whisper_backend", "qualname": "check_whisper_backend", "full_name": "examples.benchmark_audio.check_whisper_backend", "kind": "function", "signature": "def check_whisper_backend()", "parameters": [], "return_annotation": "not annotated", "docstring": "Check whether the Whisper backend can be imported.\n\nReturns:\n (available: bool, reason: str)", "summary": "Check whether the Whisper backend can be imported.", "implementation": "Function `check_whisper_backend` calls `str`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 215, "end_line": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L215-L227", "decorators": [], "calls": [ "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(True, '')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "run_tts_benchmarks", "qualname": "run_tts_benchmarks", "full_name": "examples.benchmark_audio.run_tts_benchmarks", "kind": "function", "signature": "def run_tts_benchmarks()", "parameters": [], "return_annotation": "not annotated", "docstring": "Run all TTS benchmarks.", "summary": "Run all TTS benchmarks.", "implementation": "Function `run_tts_benchmarks` calls `print`, `benchmark_tts`, `results.append`; returns `results`.", "documented": true, "public": true, "addressable": true, "line": 230, "end_line": 257, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L230-L257", "decorators": [], "calls": [ "print", "benchmark_tts", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "run_stt_benchmarks", "qualname": "run_stt_benchmarks", "full_name": "examples.benchmark_audio.run_stt_benchmarks", "kind": "function", "signature": "def run_stt_benchmarks(audio_path: str)", "parameters": [ { "name": "audio_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Run all STT benchmarks.", "summary": "Run all STT benchmarks.", "implementation": "Function `run_stt_benchmarks` calls `print`, `check_whisper_backend`, `alias.startswith`, `benchmark_stt`; returns `results`.", "documented": true, "public": true, "addressable": true, "line": 260, "end_line": 295, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L260-L295", "decorators": [], "calls": [ "print", "check_whisper_backend", "alias.startswith", "benchmark_stt", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.benchmark_audio.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "documented": false, "public": true, "addressable": true, "line": 298, "end_line": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_audio.py#L298-L328", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "generate_test_audio", "run_tts_benchmarks", "run_stt_benchmarks", "os.path.exists", "os.unlink" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.benchmark_detokenizer", "path": "examples/benchmark_detokenizer.py", "page_path": "reference/source/examples/benchmark_detokenizer.md", "docstring": "Benchmark: Streaming Detokenizer vs Naive Decode\n\nCompares performance of:\n1. Old method: tokenizer.decode([token]) for each token\n2. New method: StreamingDetokenizer.add_token() + last_segment\n\nRun:\n python examples/benchmark_detokenizer.py", "summary": "Benchmark: Streaming Detokenizer vs Naive Decode Compares performance of: 1.", "line_count": 186, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_detokenizer.py#L1-L186", "members": [ "benchmark_naive_decode", "benchmark_streaming_detokenizer", "main" ], "symbols": [ { "name": "benchmark_naive_decode", "qualname": "benchmark_naive_decode", "full_name": "examples.benchmark_detokenizer.benchmark_naive_decode", "kind": "function", "signature": "def benchmark_naive_decode(tokenizer, tokens, iterations=10)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "iterations", "kind": "positional or keyword", "annotation": "not annotated", "default": "10", "required": false, "description": "Optional positional or keyword input; defaults to `10`." } ], "return_annotation": "not annotated", "docstring": "Benchmark naive decode approach (old method).", "summary": "Benchmark naive decode approach (old method).", "implementation": "Function `benchmark_naive_decode` calls `range`, `time.perf_counter`, `tokenizer.decode`, `texts.append`; returns `{'method': 'naive_decode', 'mean_ms': statistics.mean(times) * 1000, 'std_ms': statistics.stdev(times) * 1000 if len(ti…`.", "documented": true, "public": true, "addressable": true, "line": 26, "end_line": 48, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_detokenizer.py#L26-L48", "decorators": [], "calls": [ "range", "time.perf_counter", "tokenizer.decode", "texts.append", "times.append", "statistics.mean", "len", "statistics.stdev", "min", "max" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'method': 'naive_decode', 'mean_ms': statistics.mean(times) * 1000, 'std_ms': statistics.stdev(times) * 1000 if len(ti…" ], "awaits": false, "yields": false }, { "name": "benchmark_streaming_detokenizer", "qualname": "benchmark_streaming_detokenizer", "full_name": "examples.benchmark_detokenizer.benchmark_streaming_detokenizer", "kind": "function", "signature": "def benchmark_streaming_detokenizer(tokenizer, tokens, detokenizer_class, iterations=10)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "detokenizer_class", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "iterations", "kind": "positional or keyword", "annotation": "not annotated", "default": "10", "required": false, "description": "Optional positional or keyword input; defaults to `10`." } ], "return_annotation": "not annotated", "docstring": "Benchmark streaming detokenizer approach (new method).", "summary": "Benchmark streaming detokenizer approach (new method).", "implementation": "Function `benchmark_streaming_detokenizer` calls `range`, `detokenizer_class`, `detok.reset`, `time.perf_counter`; returns `{'method': detokenizer_class.__name__, 'mean_ms': statistics.mean(times) * 1000, 'std_ms': statistics.stdev(times) * 10…`.", "documented": true, "public": true, "addressable": true, "line": 51, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_detokenizer.py#L51-L78", "decorators": [], "calls": [ "range", "detokenizer_class", "detok.reset", "time.perf_counter", "detok.add_token", "detok.finalize", "times.append", "statistics.mean", "len", "statistics.stdev", "min", "max" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'method': detokenizer_class.__name__, 'mean_ms': statistics.mean(times) * 1000, 'std_ms': statistics.stdev(times) * 10…" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.benchmark_detokenizer.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `print`, `Path`, `snapshot_download`, `load_tokenizer`.", "implementation": "Function `main` calls `print`, `Path`, `snapshot_download`, `load_tokenizer`.", "documented": false, "public": true, "addressable": true, "line": 81, "end_line": 182, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/benchmark_detokenizer.py#L81-L182", "decorators": [], "calls": [ "print", "Path", "snapshot_download", "load_tokenizer", "AutoTokenizer.from_pretrained", "type", "raw_tokenizer.encode", "len", "benchmark_naive_decode", "benchmark_streaming_detokenizer", "float", "results.append", "statistics.mean", "raw_tokenizer.decode", "tokenizer_wrapper._detokenizer_class", "detok.reset", "detok.add_token", "detok.finalize", "repr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.closed_captions", "path": "examples/closed_captions.py", "page_path": "reference/source/examples/closed_captions.md", "docstring": "Closed Captions (CC) - Real-time Subtitles\n\nUltra low-latency transcription for live subtitles/closed captions.\nSmall chunks, fast processing, continuous output.\n\nUsage:\n python examples/closed_captions.py\n python examples/closed_captions.py --language es\n\nRequirements:\n pip install sounddevice soundfile numpy", "summary": "Closed Captions (CC) - Real-time Subtitles Ultra low-latency transcription for live subtitles/closed captions.", "line_count": 166, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L1-L166", "members": [ "MODEL_ALIASES", "SAMPLE_RATE", "ClosedCaptions", "main" ], "symbols": [ { "name": "ClosedCaptions", "qualname": "ClosedCaptions", "full_name": "examples.closed_captions.ClosedCaptions", "kind": "class", "signature": "class ClosedCaptions", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "language", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "chunk_sec", "kind": "positional or keyword", "annotation": "float", "default": "1.5", "required": false, "description": "Optional positional or keyword input; defaults to `1.5`." } ], "return_annotation": "ClosedCaptions", "docstring": "Real-time closed captions.", "summary": "Real-time closed captions.", "implementation": "Class `ClosedCaptions` declares 7 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 145, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L40-L145", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ClosedCaptions.__init__", "full_name": "examples.closed_captions.ClosedCaptions.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, language: str=None, chunk_sec: float=1.5)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "language", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "chunk_sec", "kind": "positional or keyword", "annotation": "float", "default": "1.5", "required": false, "description": "Optional positional or keyword input; defaults to `1.5`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `ClosedCaptions.__init__` updates `self.model_name`, `self.language`, `self.chunk_sec`, `self.chunk_samples`; calls `int`, `queue.Queue`.", "implementation": "Method `ClosedCaptions.__init__` updates `self.model_name`, `self.language`, `self.chunk_sec`, `self.chunk_samples`; calls `int`, `queue.Queue`.", "documented": false, "public": false, "addressable": true, "line": 43, "end_line": 55, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L43-L55", "decorators": [], "calls": [ "int", "queue.Queue" ], "state_reads": [], "state_writes": [ "self.model_name", "self.language", "self.chunk_sec", "self.chunk_samples", "self.audio_queue", "self.running", "self.engine", "self.current_line", "self.lines" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load_model", "qualname": "ClosedCaptions.load_model", "full_name": "examples.closed_captions.ClosedCaptions.load_model", "kind": "method", "signature": "def load_model(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `ClosedCaptions.load_model` updates `self.engine`; calls `STTEngine`, `self.engine.load`.", "implementation": "Method `ClosedCaptions.load_model` updates `self.engine`; calls `STTEngine`, `self.engine.load`.", "documented": false, "public": true, "addressable": true, "line": 57, "end_line": 60, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L57-L60", "decorators": [], "calls": [ "STTEngine", "self.engine.load" ], "state_reads": [ "self.model_name", "self.engine.load", "self.engine" ], "state_writes": [ "self.engine" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "audio_callback", "qualname": "ClosedCaptions.audio_callback", "full_name": "examples.closed_captions.ClosedCaptions.audio_callback", "kind": "method", "signature": "def audio_callback(self, indata, frames, time_info, status)", "parameters": [ { "name": "indata", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "frames", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "time_info", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "status", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `ClosedCaptions.audio_callback` calls `self.audio_queue.put`, `indata.copy().flatten`, `indata.copy`.", "implementation": "Method `ClosedCaptions.audio_callback` calls `self.audio_queue.put`, `indata.copy().flatten`, `indata.copy`.", "documented": false, "public": true, "addressable": true, "line": 62, "end_line": 64, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L62-L64", "decorators": [], "calls": [ "self.audio_queue.put", "indata.copy().flatten", "indata.copy" ], "state_reads": [ "self.running", "self.audio_queue.put", "self.audio_queue" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "transcribe", "qualname": "ClosedCaptions.transcribe", "full_name": "examples.closed_captions.ClosedCaptions.transcribe", "kind": "method", "signature": "def transcribe(self, audio)", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `ClosedCaptions.transcribe` calls `tempfile.NamedTemporaryFile`, `sf.write`, `self.engine.transcribe`, `result.text.strip`; returns `result.text.strip()`.", "implementation": "Method `ClosedCaptions.transcribe` calls `tempfile.NamedTemporaryFile`, `sf.write`, `self.engine.transcribe`, `result.text.strip`; returns `result.text.strip()`.", "documented": false, "public": true, "addressable": true, "line": 66, "end_line": 75, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L66-L75", "decorators": [], "calls": [ "tempfile.NamedTemporaryFile", "sf.write", "self.engine.transcribe", "result.text.strip", "os.unlink" ], "state_reads": [ "self.engine.transcribe", "self.engine", "self.language" ], "state_writes": [], "raises": [], "return_expressions": [ "result.text.strip()" ], "awaits": false, "yields": false }, { "name": "display_caption", "qualname": "ClosedCaptions.display_caption", "full_name": "examples.closed_captions.ClosedCaptions.display_caption", "kind": "method", "signature": "def display_caption(self, text)", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Display caption like subtitles.", "summary": "Display caption like subtitles.", "implementation": "Method `ClosedCaptions.display_caption` calls `print`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 77, "end_line": 83, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L77-L83", "decorators": [], "calls": [ "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "process_loop", "qualname": "ClosedCaptions.process_loop", "full_name": "examples.closed_captions.ClosedCaptions.process_loop", "kind": "method", "signature": "def process_loop(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Process audio continuously.", "summary": "Process audio continuously.", "implementation": "Method `ClosedCaptions.process_loop` calls `np.array`, `self.audio_queue.get`, `np.concatenate`, `len`.", "documented": true, "public": true, "addressable": true, "line": 85, "end_line": 109, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L85-L109", "decorators": [], "calls": [ "np.array", "self.audio_queue.get", "np.concatenate", "len", "np.sqrt", "np.mean", "self.transcribe", "self.display_caption" ], "state_reads": [ "self.running", "self.audio_queue.get", "self.audio_queue", "self.chunk_samples", "self.transcribe", "self.display_caption" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "run", "qualname": "ClosedCaptions.run", "full_name": "examples.closed_captions.ClosedCaptions.run", "kind": "method", "signature": "def run(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `ClosedCaptions.run` updates `self.running`; calls `print`, `' 🎬 CLOSED CAPTIONS - Real-time Subtitles'.center`, `self.model_name.split`, `threading.Thread`.", "implementation": "Method `ClosedCaptions.run` updates `self.running`; calls `print`, `' 🎬 CLOSED CAPTIONS - Real-time Subtitles'.center`, `self.model_name.split`, `threading.Thread`.", "documented": false, "public": true, "addressable": true, "line": 111, "end_line": 145, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L111-L145", "decorators": [], "calls": [ "print", "' 🎬 CLOSED CAPTIONS - Real-time Subtitles'.center", "self.model_name.split", "threading.Thread", "processor.start", "sd.InputStream", "int", "time.sleep" ], "state_reads": [ "self.chunk_sec", "self.model_name.split", "self.model_name", "self.process_loop", "self.audio_callback" ], "state_writes": [ "self.running" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.closed_captions.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `MODEL_ALIASES.get`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `MODEL_ALIASES.get`.", "documented": false, "public": true, "addressable": true, "line": 148, "end_line": 162, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/closed_captions.py#L148-L162", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "MODEL_ALIASES.get", "print", "ClosedCaptions", "cc.load_model", "cc.run" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.demo_openai_image", "path": "examples/demo_openai_image.py", "page_path": "reference/source/examples/demo_openai_image.md", "docstring": "Demo: OpenAI API - Image Analysis\n\nShows how to use vllm-mlx with the OpenAI Python SDK for image understanding.\n\nUsage:\n 1. Start the server with a VLM model (\"vision-model\" is the name used in the OpenAI API):\n vllm-mlx serve --served-model-name vision-model mlx-community/Qwen3-VL-4B-Instruct-3bit --port 8000\n\n 2. Run this script:\n python examples/demo_openai_image.py", "summary": "Demo: OpenAI API - Image Analysis Shows how to use vllm-mlx with the OpenAI Python SDK for image understanding.", "line_count": 137, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/demo_openai_image.py#L1-L137", "members": [ "client", "image_url", "response", "scene_url", "food_url", "messages" ], "symbols": [] }, { "name": "examples.demo_openai_text", "path": "examples/demo_openai_text.py", "page_path": "reference/source/examples/demo_openai_text.md", "docstring": "Demo: OpenAI API - Text Chat\n\nShows how to use vllm-mlx with the OpenAI Python SDK for text-only chat.\n\nUsage:\n 1. Start the server with any model (served model name is defaulted to \"mlx-community/Llama-3.2-3B-Instruct-4bit\"):\n vllm-mlx serve mlx-community/Llama-3.2-3B-Instruct-4bit --port 8000\n\n 2. Run this script:\n python examples/demo_openai_text.py", "summary": "Demo: OpenAI API - Text Chat Shows how to use vllm-mlx with the OpenAI Python SDK for text-only chat.", "line_count": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/demo_openai_text.py#L1-L123", "members": [ "client", "response", "stream", "messages", "prompt", "response_low", "response_high" ], "symbols": [] }, { "name": "examples.demo_openai_video", "path": "examples/demo_openai_video.py", "page_path": "reference/source/examples/demo_openai_video.md", "docstring": "Demo: OpenAI API - Video Analysis\n\nShows how to use vllm-mlx with the OpenAI Python SDK for video understanding.\n\nUsage:\n 1. Start the server with a VLM model (\"video-model\" is the name used in the OpenAI API):\n vllm-mlx serve --served-model-name video-model mlx-community/Qwen3-VL-4B-Instruct-3bit --port 8000\n\n 2. Run this script:\n python examples/demo_openai_video.py", "summary": "Demo: OpenAI API - Video Analysis Shows how to use vllm-mlx with the OpenAI Python SDK for video understanding.", "line_count": 156, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/demo_openai_video.py#L1-L156", "members": [ "client", "video_url", "response", "action_video_url", "messages" ], "symbols": [] }, { "name": "examples.mcp_chat", "path": "examples/mcp_chat.py", "page_path": "reference/source/examples/mcp_chat.md", "docstring": "Interactive chat with MCP tools.\n\nThe LLM can use MCP tools (filesystem, etc.) to perform actions.\n\nUsage:\n python examples/mcp_chat.py\n\nExample prompts:\n - \"Create a file at /tmp/test.txt with content hello world\"\n - \"List files in /tmp\"\n - \"Read the file /tmp/test.txt\"", "summary": "Interactive chat with MCP tools.", "line_count": 187, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_chat.py#L1-L187", "members": [ "BASE_URL", "get_mcp_tools", "execute_tool", "chat", "main" ], "symbols": [ { "name": "get_mcp_tools", "qualname": "get_mcp_tools", "full_name": "examples.mcp_chat.get_mcp_tools", "kind": "function", "signature": "def get_mcp_tools()", "parameters": [], "return_annotation": "not annotated", "docstring": "Get MCP tools in OpenAI format.", "summary": "Get MCP tools in OpenAI format.", "implementation": "Function `get_mcp_tools` calls `requests.get(f'{BASE_URL}/v1/mcp/tools').json`, `requests.get`, `response.get`, `tools.append`; returns `tools`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 35, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_chat.py#L22-L35", "decorators": [], "calls": [ "requests.get(f'{BASE_URL}/v1/mcp/tools').json", "requests.get", "response.get", "tools.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tools" ], "awaits": false, "yields": false }, { "name": "execute_tool", "qualname": "execute_tool", "full_name": "examples.mcp_chat.execute_tool", "kind": "function", "signature": "def execute_tool(tool_name: str, arguments: dict)", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "arguments", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Execute an MCP tool.", "summary": "Execute an MCP tool.", "implementation": "Function `execute_tool` calls `requests.post(f'{BASE_URL}/v1/mcp/execute', json={'tool_name': tool_name, 'arguments': arguments}).json`, `requests.post`; returns `response`.", "documented": true, "public": true, "addressable": true, "line": 38, "end_line": 44, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_chat.py#L38-L44", "decorators": [], "calls": [ "requests.post(f'{BASE_URL}/v1/mcp/execute', json={'tool_name': tool_name, 'arguments': arguments}).json", "requests.post" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "response" ], "awaits": false, "yields": false }, { "name": "chat", "qualname": "chat", "full_name": "examples.mcp_chat.chat", "kind": "function", "signature": "def chat(messages: list, tools: list)", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Send message to LLM with tools.", "summary": "Send message to LLM with tools.", "implementation": "Function `chat` calls `requests.post`, `response.json`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 47, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_chat.py#L47-L66", "decorators": [], "calls": [ "requests.post", "response.json" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'error': f'HTTP {response.status_code}: {response.text[:200]}'}", "response.json()", "{'error': 'Request timed out'}", "{'error': f'Invalid JSON response: {e}'}" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.mcp_chat.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `print`, `get_mcp_tools`, `len`, `'\\n'.join`; returns `None`.", "implementation": "Function `main` calls `print`, `get_mcp_tools`, `len`, `'\\n'.join`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 69, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_chat.py#L69-L183", "decorators": [], "calls": [ "print", "get_mcp_tools", "len", "'\\n'.join", "input('\\nYou: ').strip", "input", "user_input.lower", "messages.append", "chat", "messages.pop", "response.get", "choice.get", "assistant_message.get", "json.loads", "execute_tool", "result.get", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "examples.mcp_tool_use", "path": "examples/mcp_tool_use.py", "page_path": "reference/source/examples/mcp_tool_use.md", "docstring": "Example: MCP Tool Use with vllm-mlx\n\nThis example demonstrates how to use MCP (Model Context Protocol) tools\nwith the vllm-mlx server.\n\nPrerequisites:\n1. Install MCP support: pip install vllm-mlx[mcp]\n2. Create mcp.json config (see example below)\n3. Start server with MCP: vllm-mlx serve --mcp-config mcp.json\n\nExample mcp.json:\n{\n \"servers\": {\n \"filesystem\": {\n \"transport\": \"stdio\",\n \"command\": \"npx\",\n \"args\": [\"-y\", \"@modelcontextprotocol/server-filesystem\", \"/tmp\"]\n }\n }\n}\n\nUsage:\n python examples/mcp_tool_use.py", "summary": "Example: MCP Tool Use with vllm-mlx This example demonstrates how to use MCP (Model Context Protocol) tools with the vllm-mlx server.", "line_count": 176, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_tool_use.py#L1-L176", "members": [ "main", "list_mcp_servers" ], "symbols": [ { "name": "main", "qualname": "main", "full_name": "examples.mcp_tool_use.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `OpenAI`, `print`, `requests.get(f'{base_url}/health').json`, `requests.get`; returns `None`.", "implementation": "Function `main` calls `OpenAI`, `print`, `requests.get(f'{base_url}/health').json`, `requests.get`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 35, "end_line": 159, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_tool_use.py#L35-L159", "decorators": [], "calls": [ "OpenAI", "print", "requests.get(f'{base_url}/health').json", "requests.get", "health.get", "requests.get(f'{api_base}/mcp/tools').json", "tools_response.get", "client.chat.completions.create", "len", "requests.post(f'{api_base}/mcp/execute', json={'tool_name': tool_call.function.name, 'arguments': json.loads(tool_call.…", "requests.post", "json.loads", "result.get", "str", "messages.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "list_mcp_servers", "qualname": "list_mcp_servers", "full_name": "examples.mcp_tool_use.list_mcp_servers", "kind": "function", "signature": "def list_mcp_servers()", "parameters": [], "return_annotation": "not annotated", "docstring": "Helper to list MCP server status.", "summary": "Helper to list MCP server status.", "implementation": "Function `list_mcp_servers` calls `requests.get(f'{base_url}/mcp/servers').json`, `requests.get`, `print`, `servers.get`.", "documented": true, "public": true, "addressable": true, "line": 162, "end_line": 172, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mcp_tool_use.py#L162-L172", "decorators": [], "calls": [ "requests.get(f'{base_url}/mcp/servers').json", "requests.get", "print", "servers.get", "server.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.mic_live", "path": "examples/mic_live.py", "page_path": "reference/source/examples/mic_live.md", "docstring": "Live Speech Transcription - Real-time with Voice Activity Detection\n\nTranscribes speech as you talk, detecting when you pause to process audio.\nMuch more natural than fixed-chunk transcription.\n\nUsage:\n python examples/mic_live.py\n python examples/mic_live.py --model parakeet # Faster for English\n\nRequirements:\n pip install sounddevice soundfile numpy", "summary": "Live Speech Transcription - Real-time with Voice Activity Detection Transcribes speech as you talk, detecting when you pause to process audio.", "line_count": 245, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L1-L245", "members": [ "MODEL_ALIASES", "SAMPLE_RATE", "LiveTranscriber", "main" ], "symbols": [ { "name": "LiveTranscriber", "qualname": "LiveTranscriber", "full_name": "examples.mic_live.LiveTranscriber", "kind": "class", "signature": "class LiveTranscriber", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "language", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "LiveTranscriber", "docstring": "Live transcription with voice activity detection.", "summary": "Live transcription with voice activity detection.", "implementation": "Class `LiveTranscriber` declares 7 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 42, "end_line": 201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L42-L201", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "LiveTranscriber.__init__", "full_name": "examples.mic_live.LiveTranscriber.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, language: str=None)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "language", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `LiveTranscriber.__init__` updates `self.model_name`, `self.language`, `self.silence_threshold`, `self.speech_pad_ms`; calls `deque`, `queue.Queue`.", "implementation": "Method `LiveTranscriber.__init__` updates `self.model_name`, `self.language`, `self.silence_threshold`, `self.speech_pad_ms`; calls `deque`, `queue.Queue`.", "documented": false, "public": false, "addressable": true, "line": 45, "end_line": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L45-L68", "decorators": [], "calls": [ "deque", "queue.Queue" ], "state_reads": [], "state_writes": [ "self.model_name", "self.language", "self.silence_threshold", "self.speech_pad_ms", "self.min_speech_ms", "self.silence_duration_ms", "self.audio_buffer", "self.is_speaking", "self.speech_start", "self.last_speech_time", "self.pending_audio", "self.audio_queue", "self.result_queue", "self.running", "self.engine", "self.full_transcript" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load_model", "qualname": "LiveTranscriber.load_model", "full_name": "examples.mic_live.LiveTranscriber.load_model", "kind": "method", "signature": "def load_model(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Load STT model.", "summary": "Load STT model.", "implementation": "Method `LiveTranscriber.load_model` updates `self.engine`; calls `print`, `STTEngine`, `self.engine.load`.", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 76, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L70-L76", "decorators": [], "calls": [ "print", "STTEngine", "self.engine.load" ], "state_reads": [ "self.model_name", "self.engine.load", "self.engine" ], "state_writes": [ "self.engine" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_audio_level", "qualname": "LiveTranscriber.get_audio_level", "full_name": "examples.mic_live.LiveTranscriber.get_audio_level", "kind": "method", "signature": "def get_audio_level(self, audio)", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Get RMS audio level.", "summary": "Get RMS audio level.", "implementation": "Method `LiveTranscriber.get_audio_level` calls `np.sqrt`, `np.mean`; returns `np.sqrt(np.mean(audio ** 2))`.", "documented": true, "public": true, "addressable": true, "line": 78, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L78-L80", "decorators": [], "calls": [ "np.sqrt", "np.mean" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "np.sqrt(np.mean(audio ** 2))" ], "awaits": false, "yields": false }, { "name": "audio_callback", "qualname": "LiveTranscriber.audio_callback", "full_name": "examples.mic_live.LiveTranscriber.audio_callback", "kind": "method", "signature": "def audio_callback(self, indata, frames, time_info, status)", "parameters": [ { "name": "indata", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "frames", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "time_info", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "status", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Audio input callback.", "summary": "Audio input callback.", "implementation": "Method `LiveTranscriber.audio_callback` calls `self.audio_queue.put`, `time.time`, `indata.copy().flatten`, `indata.copy`.", "documented": true, "public": true, "addressable": true, "line": 82, "end_line": 85, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L82-L85", "decorators": [], "calls": [ "self.audio_queue.put", "time.time", "indata.copy().flatten", "indata.copy" ], "state_reads": [ "self.running", "self.audio_queue.put", "self.audio_queue" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "transcribe_audio", "qualname": "LiveTranscriber.transcribe_audio", "full_name": "examples.mic_live.LiveTranscriber.transcribe_audio", "kind": "method", "signature": "def transcribe_audio(self, audio)", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Transcribe audio array.", "summary": "Transcribe audio array.", "implementation": "Method `LiveTranscriber.transcribe_audio` calls `tempfile.NamedTemporaryFile`, `sf.write`, `self.engine.transcribe`, `result.text.strip`; returns `result.text.strip()`.", "documented": true, "public": true, "addressable": true, "line": 87, "end_line": 99, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L87-L99", "decorators": [], "calls": [ "tempfile.NamedTemporaryFile", "sf.write", "self.engine.transcribe", "result.text.strip", "os.unlink" ], "state_reads": [ "self.engine.transcribe", "self.engine", "self.language" ], "state_writes": [], "raises": [], "return_expressions": [ "result.text.strip()" ], "awaits": false, "yields": false }, { "name": "process_audio_stream", "qualname": "LiveTranscriber.process_audio_stream", "full_name": "examples.mic_live.LiveTranscriber.process_audio_stream", "kind": "method", "signature": "def process_audio_stream(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Process audio with VAD.", "summary": "Process audio with VAD.", "implementation": "Method `LiveTranscriber.process_audio_stream` updates `self.is_speaking`, `self.speech_start`, `self.last_speech_time`; calls `self.audio_queue.get`, `self.get_audio_level`, `print`, `speech_buffer.extend`.", "documented": true, "public": true, "addressable": true, "line": 101, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L101-L160", "decorators": [], "calls": [ "self.audio_queue.get", "self.get_audio_level", "print", "speech_buffer.extend", "np.array", "self.transcribe_audio", "len", "self.full_transcript.append" ], "state_reads": [ "self.running", "self.audio_queue.get", "self.audio_queue", "self.get_audio_level", "self.silence_threshold", "self.is_speaking", "self.last_speech_time", "self.speech_start", "self.silence_duration_ms", "self.min_speech_ms", "self.transcribe_audio", "self.full_transcript.append", "self.full_transcript" ], "state_writes": [ "self.is_speaking", "self.speech_start", "self.last_speech_time" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "run", "qualname": "LiveTranscriber.run", "full_name": "examples.mic_live.LiveTranscriber.run", "kind": "method", "signature": "def run(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Start live transcription.", "summary": "Start live transcription.", "implementation": "Method `LiveTranscriber.run` updates `self.running`; calls `print`, `threading.Thread`, `process_thread.start`, `sd.InputStream`; returns `self.full_transcript`.", "documented": true, "public": true, "addressable": true, "line": 162, "end_line": 201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L162-L201", "decorators": [], "calls": [ "print", "threading.Thread", "process_thread.start", "sd.InputStream", "int", "time.sleep", "process_thread.join" ], "state_reads": [ "self.process_audio_stream", "self.audio_callback", "self.full_transcript" ], "state_writes": [ "self.running" ], "raises": [], "return_expressions": [ "self.full_transcript" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.mic_live.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "documented": false, "public": true, "addressable": true, "line": 204, "end_line": 241, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_live.py#L204-L241", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "MODEL_ALIASES.get", "LiveTranscriber", "transcriber.load_model", "transcriber.run", "' '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.mic_realtime", "path": "examples/mic_realtime.py", "page_path": "reference/source/examples/mic_realtime.md", "docstring": "Real-Time Microphone Transcription with Whisper - vllm-mlx\n\nTranscribes speech in real-time as you speak using your Mac's microphone.\n\nUsage:\n python examples/mic_realtime.py # Default (3s chunks)\n python examples/mic_realtime.py --chunk 5 # 5 second chunks\n python examples/mic_realtime.py --model parakeet # Faster model\n\nRequirements:\n pip install sounddevice soundfile numpy", "summary": "Real-Time Microphone Transcription with Whisper - vllm-mlx Transcribes speech in real-time as you speak using your Mac's microphone.", "line_count": 236, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L1-L236", "members": [ "MODEL_ALIASES", "SAMPLE_RATE", "CHANNELS", "RealtimeTranscriber", "main" ], "symbols": [ { "name": "RealtimeTranscriber", "qualname": "RealtimeTranscriber", "full_name": "examples.mic_realtime.RealtimeTranscriber", "kind": "class", "signature": "class RealtimeTranscriber", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "chunk_duration", "kind": "positional or keyword", "annotation": "float", "default": "3.0", "required": false, "description": "Optional positional or keyword input; defaults to `3.0`." }, { "name": "language", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "RealtimeTranscriber", "docstring": "Real-time audio transcription using Whisper.", "summary": "Real-time audio transcription using Whisper.", "implementation": "Class `RealtimeTranscriber` declares 6 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L45-L171", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "RealtimeTranscriber.__init__", "full_name": "examples.mic_realtime.RealtimeTranscriber.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, chunk_duration: float=3.0, language: str=None)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "chunk_duration", "kind": "positional or keyword", "annotation": "float", "default": "3.0", "required": false, "description": "Optional positional or keyword input; defaults to `3.0`." }, { "name": "language", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `RealtimeTranscriber.__init__` updates `self.model_name`, `self.chunk_duration`, `self.language`, `self.sample_rate`; calls `queue.Queue`.", "implementation": "Method `RealtimeTranscriber.__init__` updates `self.model_name`, `self.chunk_duration`, `self.language`, `self.sample_rate`; calls `queue.Queue`.", "documented": false, "public": false, "addressable": true, "line": 48, "end_line": 60, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L48-L60", "decorators": [], "calls": [ "queue.Queue" ], "state_reads": [], "state_writes": [ "self.model_name", "self.chunk_duration", "self.language", "self.sample_rate", "self.audio_queue", "self.is_recording", "self.engine", "self.transcriptions" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load_model", "qualname": "RealtimeTranscriber.load_model", "full_name": "examples.mic_realtime.RealtimeTranscriber.load_model", "kind": "method", "signature": "def load_model(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Load the STT model.", "summary": "Load the STT model.", "implementation": "Method `RealtimeTranscriber.load_model` updates `self.engine`; calls `print`, `STTEngine`, `self.engine.load`.", "documented": true, "public": true, "addressable": true, "line": 62, "end_line": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L62-L68", "decorators": [], "calls": [ "print", "STTEngine", "self.engine.load" ], "state_reads": [ "self.model_name", "self.engine.load", "self.engine" ], "state_writes": [ "self.engine" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "audio_callback", "qualname": "RealtimeTranscriber.audio_callback", "full_name": "examples.mic_realtime.RealtimeTranscriber.audio_callback", "kind": "method", "signature": "def audio_callback(self, indata, frames, time_info, status)", "parameters": [ { "name": "indata", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "frames", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "time_info", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "status", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Callback for audio input stream.", "summary": "Callback for audio input stream.", "implementation": "Method `RealtimeTranscriber.audio_callback` calls `print`, `self.audio_queue.put`, `indata.copy`.", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 75, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L70-L75", "decorators": [], "calls": [ "print", "self.audio_queue.put", "indata.copy" ], "state_reads": [ "self.is_recording", "self.audio_queue.put", "self.audio_queue" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "transcribe_chunk", "qualname": "RealtimeTranscriber.transcribe_chunk", "full_name": "examples.mic_realtime.RealtimeTranscriber.transcribe_chunk", "kind": "method", "signature": "def transcribe_chunk(self, audio_data)", "parameters": [ { "name": "audio_data", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Transcribe a chunk of audio.", "summary": "Transcribe a chunk of audio.", "implementation": "Method `RealtimeTranscriber.transcribe_chunk` calls `tempfile.NamedTemporaryFile`, `sf.write`, `self.engine.transcribe`, `result.text.strip`; returns `result.text.strip()`.", "documented": true, "public": true, "addressable": true, "line": 77, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L77-L90", "decorators": [], "calls": [ "tempfile.NamedTemporaryFile", "sf.write", "self.engine.transcribe", "result.text.strip", "os.unlink" ], "state_reads": [ "self.sample_rate", "self.engine.transcribe", "self.engine", "self.language" ], "state_writes": [], "raises": [], "return_expressions": [ "result.text.strip()" ], "awaits": false, "yields": false }, { "name": "process_audio", "qualname": "RealtimeTranscriber.process_audio", "full_name": "examples.mic_realtime.RealtimeTranscriber.process_audio", "kind": "method", "signature": "def process_audio(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Process audio chunks in real-time.", "summary": "Process audio chunks in real-time.", "implementation": "Method `RealtimeTranscriber.process_audio` calls `int`, `np.array`, `self.audio_queue.empty`, `self.audio_queue.get`.", "documented": true, "public": true, "addressable": true, "line": 92, "end_line": 129, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L92-L129", "decorators": [], "calls": [ "int", "np.array", "self.audio_queue.empty", "self.audio_queue.get", "np.concatenate", "data.flatten", "len", "np.abs(chunk).max", "np.abs", "self.transcribe_chunk", "self.transcriptions.append", "print", "np.abs(buffer).max" ], "state_reads": [ "self.chunk_duration", "self.sample_rate", "self.is_recording", "self.audio_queue.empty", "self.audio_queue", "self.audio_queue.get", "self.transcribe_chunk", "self.transcriptions.append", "self.transcriptions" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "run", "qualname": "RealtimeTranscriber.run", "full_name": "examples.mic_realtime.RealtimeTranscriber.run", "kind": "method", "signature": "def run(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Start real-time transcription.", "summary": "Start real-time transcription.", "implementation": "Method `RealtimeTranscriber.run` updates `self.is_recording`; calls `print`, `sd.InputStream`, `int`, `threading.Thread`; returns `self.transcriptions`.", "documented": true, "public": true, "addressable": true, "line": 131, "end_line": 171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L131-L171", "decorators": [], "calls": [ "print", "sd.InputStream", "int", "threading.Thread", "process_thread.start", "time.sleep", "process_thread.join" ], "state_reads": [ "self.chunk_duration", "self.sample_rate", "self.audio_callback", "self.process_audio", "self.transcriptions" ], "state_writes": [ "self.is_recording" ], "raises": [], "return_expressions": [ "self.transcriptions" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.mic_realtime.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 174, "end_line": 232, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_realtime.py#L174-L232", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "MODEL_ALIASES.items", "MODEL_ALIASES.get", "RealtimeTranscriber", "transcriber.load_model", "transcriber.run", "' '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "examples.mic_transcribe", "path": "examples/mic_transcribe.py", "page_path": "reference/source/examples/mic_transcribe.md", "docstring": "Live Microphone Transcription with Whisper - vllm-mlx\n\nRecords audio from your Mac's microphone and transcribes it using Whisper.\n\nUsage:\n python examples/mic_transcribe.py # Record until Enter\n python examples/mic_transcribe.py --duration 5 # Record for 5 seconds\n python examples/mic_transcribe.py --model whisper-small # Use smaller model\n python examples/mic_transcribe.py --continuous # Continuous mode\n\nRequirements:\n pip install sounddevice soundfile", "summary": "Live Microphone Transcription with Whisper - vllm-mlx Records audio from your Mac's microphone and transcribes it using Whisper.", "line_count": 221, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_transcribe.py#L1-L221", "members": [ "MODEL_ALIASES", "record_audio", "save_audio", "main" ], "symbols": [ { "name": "record_audio", "qualname": "record_audio", "full_name": "examples.mic_transcribe.record_audio", "kind": "function", "signature": "def record_audio(duration=None, sample_rate=16000)", "parameters": [ { "name": "duration", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Recording duration in seconds. If None, records until Enter." }, { "name": "sample_rate", "kind": "positional or keyword", "annotation": "not annotated", "default": "16000", "required": false, "description": "Audio sample rate (16000 Hz for Whisper)" } ], "return_annotation": "not annotated", "docstring": "Record audio from microphone.\n\nArgs:\n duration: Recording duration in seconds. If None, records until Enter.\n sample_rate: Audio sample rate (16000 Hz for Whisper)\n\nReturns:\n numpy array of audio data", "summary": "Record audio from microphone.", "implementation": "Function `record_audio` calls `print`, `sd.rec`, `int`, `sd.wait`; returns `(audio.flatten(), sample_rate)`.", "documented": true, "public": true, "addressable": true, "line": 39, "end_line": 93, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_transcribe.py#L39-L93", "decorators": [], "calls": [ "print", "sd.rec", "int", "sd.wait", "threading.Event", "threading.Thread", "enter_thread.start", "stop_recording.is_set", "chunks.append", "len", "np.concatenate", "np.array", "audio.flatten" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(audio.flatten(), sample_rate)" ], "awaits": false, "yields": false }, { "name": "wait_for_enter", "qualname": "record_audio.wait_for_enter", "full_name": "examples.mic_transcribe.record_audio.wait_for_enter", "kind": "nested function", "signature": "def wait_for_enter()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `record_audio.wait_for_enter` calls `input`, `stop_recording.set`.", "implementation": "Nested Function `record_audio.wait_for_enter` calls `input`, `stop_recording.set`.", "documented": false, "public": false, "addressable": false, "line": 71, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_transcribe.py#L71-L73", "decorators": [], "calls": [ "input", "stop_recording.set" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "save_audio", "qualname": "save_audio", "full_name": "examples.mic_transcribe.save_audio", "kind": "function", "signature": "def save_audio(audio, sample_rate, path)", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "sample_rate", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "path", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Save audio to WAV file.", "summary": "Save audio to WAV file.", "implementation": "Function `save_audio` calls `sf.write`.", "documented": true, "public": true, "addressable": true, "line": 96, "end_line": 99, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_transcribe.py#L96-L99", "decorators": [], "calls": [ "sf.write" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.mic_transcribe.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 102, "end_line": 217, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mic_transcribe.py#L102-L217", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "sd.query_devices", "MODEL_ALIASES.items", "MODEL_ALIASES.get", "STTEngine", "engine.load", "record_audio", "len", "tempfile.NamedTemporaryFile", "save_audio", "engine.transcribe", "os.unlink" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "examples.mllm_benchmark", "path": "examples/mllm_benchmark.py", "page_path": "reference/source/examples/mllm_benchmark.md", "docstring": "MLLM Benchmark Script for vllm-mlx\n\nTests Multimodal Language Models with real images of dogs from Wikimedia Commons\nat different resolutions and measures performance metrics.\n\nUsage:\n # Start the MLLM server first:\n python -m vllm_mlx.server --model mlx-community/Qwen3-VL-4B-Instruct-3bit --port 8000\n\n # Run benchmark:\n python examples/mllm_benchmark.py\n\n # Or specify server URL:\n python examples/mllm_benchmark.py --server-url http://localhost:8000", "summary": "MLLM Benchmark Script for vllm-mlx Tests Multimodal Language Models with real images of dogs from Wikimedia Commons at different resolutions and measures performance metrics.", "line_count": 443, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L1-L443", "members": [ "TEST_IMAGES", "PRIMARY_DOG_IMAGE", "BenchmarkResult", "download_image", "resize_image", "image_to_base64", "run_mllm_request", "benchmark_resolution", "run_benchmark", "print_results", "save_results", "main" ], "symbols": [ { "name": "BenchmarkResult", "qualname": "BenchmarkResult", "full_name": "examples.mllm_benchmark.BenchmarkResult", "kind": "class", "signature": "class BenchmarkResult", "parameters": [ { "name": "resolution", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "width", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "height", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "pixels", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "time_seconds", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens_generated", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens_per_second", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "response_preview", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "BenchmarkResult", "docstring": "Result from a single benchmark run.", "summary": "Result from a single benchmark run.", "implementation": "Class `BenchmarkResult` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 65, "end_line": 74, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L65-L74", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "download_image", "qualname": "download_image", "full_name": "examples.mllm_benchmark.download_image", "kind": "function", "signature": "def download_image(url: str, timeout: int=30) -> Image.Image", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "30", "required": false, "description": "Optional positional or keyword input; defaults to `30`." } ], "return_annotation": "Image.Image", "docstring": "Download image from URL and return PIL Image.", "summary": "Download image from URL and return PIL Image.", "implementation": "Function `download_image` calls `requests.get`, `response.raise_for_status`, `Image.open`, `io.BytesIO`; returns `Image.open(io.BytesIO(response.content))`.", "documented": true, "public": true, "addressable": true, "line": 77, "end_line": 84, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L77-L84", "decorators": [], "calls": [ "requests.get", "response.raise_for_status", "Image.open", "io.BytesIO" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Image.open(io.BytesIO(response.content))" ], "awaits": false, "yields": false }, { "name": "resize_image", "qualname": "resize_image", "full_name": "examples.mllm_benchmark.resize_image", "kind": "function", "signature": "def resize_image(img: Image.Image, width: int, height: int) -> Image.Image", "parameters": [ { "name": "img", "kind": "positional or keyword", "annotation": "Image.Image", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "width", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "height", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Image.Image", "docstring": "Resize image to specified dimensions.", "summary": "Resize image to specified dimensions.", "implementation": "Function `resize_image` calls `img.resize`; returns `img.resize((width, height), Image.Resampling.LANCZOS)`.", "documented": true, "public": true, "addressable": true, "line": 87, "end_line": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L87-L89", "decorators": [], "calls": [ "img.resize" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "img.resize((width, height), Image.Resampling.LANCZOS)" ], "awaits": false, "yields": false }, { "name": "image_to_base64", "qualname": "image_to_base64", "full_name": "examples.mllm_benchmark.image_to_base64", "kind": "function", "signature": "def image_to_base64(img: Image.Image, format: str='JPEG') -> str", "parameters": [ { "name": "img", "kind": "positional or keyword", "annotation": "Image.Image", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "format", "kind": "positional or keyword", "annotation": "str", "default": "'JPEG'", "required": false, "description": "Optional positional or keyword input; defaults to `'JPEG'`." } ], "return_annotation": "str", "docstring": "Convert PIL Image to base64 data URL.", "summary": "Convert PIL Image to base64 data URL.", "implementation": "Function `image_to_base64` calls `Image.new`, `background.paste`, `img.split`, `img.convert`; returns `f'data:{mime};base64,{b64}'`.", "documented": true, "public": true, "addressable": true, "line": 92, "end_line": 106, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L92-L106", "decorators": [], "calls": [ "Image.new", "background.paste", "img.split", "img.convert", "io.BytesIO", "img.save", "base64.b64encode(buffer.getvalue()).decode", "base64.b64encode", "buffer.getvalue" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'data:{mime};base64,{b64}'" ], "awaits": false, "yields": false }, { "name": "run_mllm_request", "qualname": "run_mllm_request", "full_name": "examples.mllm_benchmark.run_mllm_request", "kind": "function", "signature": "def run_mllm_request(server_url: str, image_b64: str, prompt: str='Describe this image in detail. What do you see?', max_tokens: int=256, model: str='default') -> tuple[str, float, int]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "image_b64", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "'Describe this image in detail. What do you see?'", "required": false, "description": "Optional positional or keyword input; defaults to `'Describe this image in detail. What do you see?'`." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Optional positional or keyword input; defaults to `256`." }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "'default'", "required": false, "description": "Optional positional or keyword input; defaults to `'default'`." } ], "return_annotation": "tuple[str, float, int]", "docstring": "Send an MLLM request to the server.\n\nReturns:\n (response_text, time_seconds, tokens_generated)", "summary": "Send an MLLM request to the server.", "implementation": "Function `run_mllm_request` calls `time.perf_counter`, `requests.post`, `response.raise_for_status`, `response.json`; returns `(text, elapsed, tokens)`.", "documented": true, "public": true, "addressable": true, "line": 109, "end_line": 153, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L109-L153", "decorators": [], "calls": [ "time.perf_counter", "requests.post", "response.raise_for_status", "response.json", "data.get('usage', {}).get", "data.get", "len", "text.split" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(text, elapsed, tokens)" ], "awaits": false, "yields": false }, { "name": "benchmark_resolution", "qualname": "benchmark_resolution", "full_name": "examples.mllm_benchmark.benchmark_resolution", "kind": "function", "signature": "def benchmark_resolution(server_url: str, base_image: Image.Image, width: int, height: int, model: str, warmup: bool=False) -> BenchmarkResult", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "base_image", "kind": "positional or keyword", "annotation": "Image.Image", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "width", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "height", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "warmup", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "BenchmarkResult", "docstring": "Run benchmark for a specific resolution.", "summary": "Run benchmark for a specific resolution.", "implementation": "Function `benchmark_resolution` calls `resize_image`, `image_to_base64`, `print`, `run_mllm_request`; returns `BenchmarkResult(resolution=resolution_name, width=width, height=height, pixels=pixels, time_seconds=elapsed, tokens_gen…`.", "documented": true, "public": true, "addressable": true, "line": 156, "end_line": 198, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L156-L198", "decorators": [], "calls": [ "resize_image", "image_to_base64", "print", "run_mllm_request", "BenchmarkResult", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "BenchmarkResult(resolution=resolution_name, width=width, height=height, pixels=pixels, time_seconds=elapsed, tokens_gen…" ], "awaits": false, "yields": false }, { "name": "run_benchmark", "qualname": "run_benchmark", "full_name": "examples.mllm_benchmark.run_benchmark", "kind": "function", "signature": "def run_benchmark(server_url: str='http://localhost:8000', resolutions: list[tuple[int, int]]=None, warmup_runs: int=1, image_url: str=None) -> list[BenchmarkResult]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "'http://localhost:8000'", "required": false, "description": "URL of the vllm-mlx server" }, { "name": "resolutions", "kind": "positional or keyword", "annotation": "list[tuple[int, int]]", "default": "None", "required": false, "description": "List of (width, height) tuples to test" }, { "name": "warmup_runs", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Number of warmup runs before measuring" }, { "name": "image_url", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "URL of image to use (default: dog from Wikimedia)" } ], "return_annotation": "list[BenchmarkResult]", "docstring": "Run full MLLM benchmark across multiple resolutions.\n\nArgs:\n server_url: URL of the vllm-mlx server\n resolutions: List of (width, height) tuples to test\n warmup_runs: Number of warmup runs before measuring\n image_url: URL of image to use (default: dog from Wikimedia)\n\nReturns:\n List of BenchmarkResult objects", "summary": "Run full MLLM benchmark across multiple resolutions.", "implementation": "Function `run_benchmark` calls `print`, `requests.get`, `health.raise_for_status`, `health.json`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 201, "end_line": 293, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L201-L293", "decorators": [], "calls": [ "print", "requests.get", "health.raise_for_status", "health.json", "health_data.get", "download_image", "range", "benchmark_resolution", "len", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "results" ], "awaits": false, "yields": false }, { "name": "print_results", "qualname": "print_results", "full_name": "examples.mllm_benchmark.print_results", "kind": "function", "signature": "def print_results(results: list[BenchmarkResult])", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[BenchmarkResult]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print benchmark results in a nice table.", "summary": "Print benchmark results in a nice table.", "implementation": "Function `print_results` calls `print`, `table_data.append`, `tabulate`, `sum`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 296, "end_line": 338, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L296-L338", "decorators": [], "calls": [ "print", "table_data.append", "tabulate", "sum", "min", "max" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "save_results", "qualname": "save_results", "full_name": "examples.mllm_benchmark.save_results", "kind": "function", "signature": "def save_results(results: list[BenchmarkResult], output_path: str)", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[BenchmarkResult]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "output_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Save benchmark results to JSON file.", "summary": "Save benchmark results to JSON file.", "implementation": "Function `save_results` calls `time.strftime`, `open`, `json.dump`, `print`.", "documented": true, "public": true, "addressable": true, "line": 341, "end_line": 364, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L341-L364", "decorators": [], "calls": [ "time.strftime", "open", "json.dump", "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.mllm_benchmark.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `run_benchmark`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `run_benchmark`.", "documented": false, "public": true, "addressable": true, "line": 367, "end_line": 439, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_benchmark.py#L367-L439", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "run_benchmark", "print_results", "save_results" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.mllm_example", "path": "examples/mllm_example.py", "page_path": "reference/source/examples/mllm_example.md", "docstring": "Multimodal Language Model (MLLM) example using vllm-mlx.\n\nThis example demonstrates multimodal inference on Apple Silicon,\nincluding image understanding and visual question answering.", "summary": "Multimodal Language Model (MLLM) example using vllm-mlx.", "line_count": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_example.py#L1-L89", "members": [ "main" ], "symbols": [ { "name": "main", "qualname": "main", "full_name": "examples.mllm_example.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `print`, `MLXMultimodalLM`, `mllm.load`, `len`; returns `None`.", "implementation": "Function `main` calls `print`, `MLXMultimodalLM`, `mllm.load`, `len`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 16, "end_line": 85, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/mllm_example.py#L16-L85", "decorators": [], "calls": [ "print", "MLXMultimodalLM", "mllm.load", "len", "mllm.generate", "Path(image_path).exists", "Path", "sys.exit", "mllm.describe_image", "mllm.answer_about_image" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "examples.simple_generate", "path": "examples/simple_generate.py", "page_path": "reference/source/examples/simple_generate.md", "docstring": "Simple text generation example using vllm-mlx.\n\nThis example demonstrates basic LLM inference on Apple Silicon\nusing the MLX backend.", "summary": "Simple text generation example using vllm-mlx.", "line_count": 71, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/simple_generate.py#L1-L71", "members": [ "main" ], "symbols": [ { "name": "main", "qualname": "main", "full_name": "examples.simple_generate.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `print`, `MLXLanguageModel`, `model.load`, `model.generate`.", "implementation": "Function `main` calls `print`, `MLXLanguageModel`, `model.load`, `model.generate`.", "documented": false, "public": true, "addressable": true, "line": 13, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/simple_generate.py#L13-L67", "decorators": [], "calls": [ "print", "MLXLanguageModel", "model.load", "model.generate", "model.stream_generate", "model.chat" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.test_batch_sync", "path": "examples/test_batch_sync.py", "page_path": "reference/source/examples/test_batch_sync.md", "docstring": "Test generate_batch_sync() performance.", "summary": "Test generate_batch_sync() performance.", "line_count": 105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batch_sync.py#L1-L105", "members": [ "main" ], "symbols": [ { "name": "main", "qualname": "main", "full_name": "examples.test_batch_sync.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `print`, `load`, `SamplingParams`, `EngineConfig`.", "implementation": "Function `main` calls `print`, `load`, `SamplingParams`, `EngineConfig`.", "documented": false, "public": true, "addressable": true, "line": 8, "end_line": 101, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batch_sync.py#L8-L101", "decorators": [], "calls": [ "print", "load", "SamplingParams", "EngineConfig", "SchedulerConfig", "EngineCore", "format_prompt", "time.perf_counter", "engine.generate_batch_sync", "sum", "len", "asyncio.run", "run_async" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "format_prompt", "qualname": "main.format_prompt", "full_name": "examples.test_batch_sync.main.format_prompt", "kind": "nested function", "signature": "def format_prompt(p)", "parameters": [ { "name": "p", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `main.format_prompt` calls `tokenizer.apply_chat_template`; returns `tokenizer.apply_chat_template([{'role': 'user', 'content': p}], tokenize=False, add_generation_prompt=True)`.", "implementation": "Nested Function `main.format_prompt` calls `tokenizer.apply_chat_template`; returns `tokenizer.apply_chat_template([{'role': 'user', 'content': p}], tokenize=False, add_generation_prompt=True)`.", "documented": false, "public": false, "addressable": false, "line": 24, "end_line": 29, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batch_sync.py#L24-L29", "decorators": [], "calls": [ "tokenizer.apply_chat_template" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tokenizer.apply_chat_template([{'role': 'user', 'content': p}], tokenize=False, add_generation_prompt=True)" ], "awaits": false, "yields": false }, { "name": "run_async", "qualname": "main.run_async", "full_name": "examples.test_batch_sync.main.run_async", "kind": "nested function", "signature": "async def run_async()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `main.run_async` calls `EngineConfig`, `SchedulerConfig`, `EngineCore`, `engine.start`; awaits asynchronous work; returns `(total_tokens, elapsed)`.", "implementation": "Nested Function `main.run_async` calls `EngineConfig`, `SchedulerConfig`, `EngineCore`, `engine.start`; awaits asynchronous work; returns `(total_tokens, elapsed)`.", "documented": false, "public": false, "addressable": false, "line": 71, "end_line": 93, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batch_sync.py#L71-L93", "decorators": [], "calls": [ "EngineConfig", "SchedulerConfig", "EngineCore", "engine.start", "format_prompt", "time.perf_counter", "engine.generate", "asyncio.gather", "sum", "engine.stop" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(total_tokens, elapsed)" ], "awaits": true, "yields": false } ] }, { "name": "examples.test_batching", "path": "examples/test_batching.py", "page_path": "reference/source/examples/test_batching.md", "docstring": "Example: Test continuous batching with vllm-mlx.\n\nThis script demonstrates the continuous batching capability by sending\nmultiple concurrent requests and measuring throughput.\n\nUsage:\n python examples/test_batching.py\n python examples/test_batching.py --model mlx-community/Qwen2.5-3B-Instruct-4bit\n python examples/test_batching.py --num-requests 10 --max-tokens 50", "summary": "Example: Test continuous batching with vllm-mlx.", "line_count": 195, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batching.py#L1-L195", "members": [ "run_single_request", "run_concurrent_requests", "print_results", "main" ], "symbols": [ { "name": "run_single_request", "qualname": "run_single_request", "full_name": "examples.test_batching.run_single_request", "kind": "function", "signature": "async def run_single_request(engine: AsyncEngineCore, request_id: str, prompt: str, sampling_params: SamplingParams) -> dict", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "AsyncEngineCore", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "SamplingParams", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Run a single request and collect timing.", "summary": "Run a single request and collect timing.", "implementation": "Function `run_single_request` calls `time.perf_counter`, `engine.add_request`, `engine.stream_outputs`, `tokens.extend`; awaits asynchronous work; returns `{'request_id': request_id, 'prompt_length': len(prompt.split()), 'num_tokens': len(tokens), 'ttft': ttft, 'total_time':…`.", "documented": true, "public": true, "addressable": true, "line": 29, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batching.py#L29-L65", "decorators": [], "calls": [ "time.perf_counter", "engine.add_request", "engine.stream_outputs", "tokens.extend", "len", "prompt.split" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'request_id': request_id, 'prompt_length': len(prompt.split()), 'num_tokens': len(tokens), 'ttft': ttft, 'total_time':…" ], "awaits": true, "yields": false }, { "name": "run_concurrent_requests", "qualname": "run_concurrent_requests", "full_name": "examples.test_batching.run_concurrent_requests", "kind": "function", "signature": "async def run_concurrent_requests(engine: AsyncEngineCore, prompts: List[str], sampling_params: SamplingParams) -> List[dict]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "AsyncEngineCore", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompts", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "SamplingParams", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "List[dict]", "docstring": "Run multiple requests concurrently.", "summary": "Run multiple requests concurrently.", "implementation": "Function `run_concurrent_requests` calls `enumerate`, `run_single_request`, `tasks.append`, `asyncio.gather`; awaits asynchronous work; returns `await asyncio.gather(*tasks)`.", "documented": true, "public": true, "addressable": true, "line": 68, "end_line": 79, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batching.py#L68-L79", "decorators": [], "calls": [ "enumerate", "run_single_request", "tasks.append", "asyncio.gather" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "await asyncio.gather(*tasks)" ], "awaits": true, "yields": false }, { "name": "print_results", "qualname": "print_results", "full_name": "examples.test_batching.print_results", "kind": "function", "signature": "def print_results(results: List[dict], total_time: float)", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "List[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "total_time", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print benchmark results.", "summary": "Print benchmark results.", "implementation": "Function `print_results` calls `print`, `sum`, `len`.", "documented": true, "public": true, "addressable": true, "line": 82, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batching.py#L82-L113", "decorators": [], "calls": [ "print", "sum", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.test_batching.main", "kind": "function", "signature": "async def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; awaits asynchronous work.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; awaits asynchronous work.", "documented": false, "public": true, "addressable": true, "line": 116, "end_line": 191, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_batching.py#L116-L191", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "load", "SchedulerConfig", "EngineConfig", "SamplingParams", "len", "AsyncEngineCore", "asyncio.sleep", "time.perf_counter", "run_concurrent_requests", "print_results" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false } ] }, { "name": "examples.test_openai_compatibility", "path": "examples/test_openai_compatibility.py", "page_path": "reference/source/examples/test_openai_compatibility.md", "docstring": "OpenAI API Compatibility Test Script for vllm-mlx.\n\nThis script tests the OpenAI API compatibility of the vllm-mlx server.\nIt tests both the direct HTTP API and the official OpenAI Python client.\n\nUsage:\n # First start the server:\n vllm-mlx serve --served-model-name default mlx-community/Qwen3-VL-4B-Instruct-3bit --port 8000\n\n # Then run this script:\n python examples/test_openai_compatibility.py\n\n # With a different server URL:\n python examples/test_openai_compatibility.py --server-url http://localhost:9000\n\n # Test only specific endpoints:\n python examples/test_openai_compatibility.py --test-image --test-video", "summary": "OpenAI API Compatibility Test Script for vllm-mlx.", "line_count": 739, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L1-L739", "members": [ "GREEN", "RED", "YELLOW", "BLUE", "RESET", "BOLD", "print_header", "print_test", "print_warning", "create_test_image", "test_health_endpoint", "test_models_endpoint", "test_chat_completions_http", "test_chat_completions_openai", "test_completions_endpoint", "test_image_chat_http", "test_image_chat_openai", "test_image_url_http", "test_streaming_chat", "create_test_video", "test_video_chat_http", "test_video_chat_openai", "test_video_url_http", "run_all_tests", "main" ], "symbols": [ { "name": "print_header", "qualname": "print_header", "full_name": "examples.test_openai_compatibility.print_header", "kind": "function", "signature": "def print_header(text: str)", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print a section header.", "summary": "Print a section header.", "implementation": "Function `print_header` calls `print`.", "documented": true, "public": true, "addressable": true, "line": 37, "end_line": 41, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L37-L41", "decorators": [], "calls": [ "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "print_test", "qualname": "print_test", "full_name": "examples.test_openai_compatibility.print_test", "kind": "function", "signature": "def print_test(name: str, passed: bool, message: str='')", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "passed", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "message", "kind": "positional or keyword", "annotation": "str", "default": "''", "required": false, "description": "Optional positional or keyword input; defaults to `''`." } ], "return_annotation": "not annotated", "docstring": "Print test result.", "summary": "Print test result.", "implementation": "Function `print_test` calls `print`.", "documented": true, "public": true, "addressable": true, "line": 44, "end_line": 49, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L44-L49", "decorators": [], "calls": [ "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "print_warning", "qualname": "print_warning", "full_name": "examples.test_openai_compatibility.print_warning", "kind": "function", "signature": "def print_warning(text: str)", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print a warning message.", "summary": "Print a warning message.", "implementation": "Function `print_warning` calls `print`.", "documented": true, "public": true, "addressable": true, "line": 52, "end_line": 54, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L52-L54", "decorators": [], "calls": [ "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "create_test_image", "qualname": "create_test_image", "full_name": "examples.test_openai_compatibility.create_test_image", "kind": "function", "signature": "def create_test_image() -> tuple[str, bytes]", "parameters": [], "return_annotation": "tuple[str, bytes]", "docstring": "Create a simple test image and return (path, bytes).", "summary": "Create a simple test image and return (path, bytes).", "implementation": "Function `create_test_image` calls `Image.new`, `io.BytesIO`, `img.save`, `buffer.getvalue`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 57, "end_line": 97, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L57-L97", "decorators": [], "calls": [ "Image.new", "io.BytesIO", "img.save", "buffer.getvalue", "tempfile.NamedTemporaryFile", "temp_file.write", "temp_file.close", "print_warning", "bytes" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(temp_file.name, img_bytes)", "(temp_file.name, minimal_png)" ], "awaits": false, "yields": false }, { "name": "test_health_endpoint", "qualname": "test_health_endpoint", "full_name": "examples.test_openai_compatibility.test_health_endpoint", "kind": "function", "signature": "def test_health_endpoint(server_url: str) -> bool", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Test the /health endpoint.", "summary": "Test the /health endpoint.", "implementation": "Function `test_health_endpoint` calls `requests.get`, `print_warning`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 100, "end_line": 109, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L100-L109", "decorators": [], "calls": [ "requests.get", "print_warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "response.status_code == 200", "False" ], "awaits": false, "yields": false }, { "name": "test_models_endpoint", "qualname": "test_models_endpoint", "full_name": "examples.test_openai_compatibility.test_models_endpoint", "kind": "function", "signature": "def test_models_endpoint(server_url: str) -> bool", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Test the /v1/models endpoint.", "summary": "Test the /v1/models endpoint.", "implementation": "Function `test_models_endpoint` calls `requests.get`, `response.json`, `isinstance`, `print_warning`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 112, "end_line": 126, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L112-L126", "decorators": [], "calls": [ "requests.get", "response.json", "isinstance", "print_warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "'data' in data and isinstance(data['data'], list)" ], "awaits": false, "yields": false }, { "name": "test_chat_completions_http", "qualname": "test_chat_completions_http", "full_name": "examples.test_openai_compatibility.test_chat_completions_http", "kind": "function", "signature": "def test_chat_completions_http(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test /v1/chat/completions with direct HTTP.", "summary": "Test /v1/chat/completions with direct HTTP.", "implementation": "Function `test_chat_completions_http` calls `requests.post`, `response.json`, `len`, `str`; has 7 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 129, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L129-L170", "decorators": [], "calls": [ "requests.post", "response.json", "len", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, f'Status code: {response.status_code}')", "(False, \"Missing 'choices' in response\")", "(False, 'Empty choices array')", "(False, \"Missing 'message' in choice\")", "(False, \"Missing 'content' in message\")", "(True, f'Response: {content[:50]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_chat_completions_openai", "qualname": "test_chat_completions_openai", "full_name": "examples.test_openai_compatibility.test_chat_completions_openai", "kind": "function", "signature": "def test_chat_completions_openai(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test /v1/chat/completions with OpenAI Python client.", "summary": "Test /v1/chat/completions with OpenAI Python client.", "implementation": "Function `test_chat_completions_openai` calls `OpenAI`, `client.chat.completions.create`, `str`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 173, "end_line": 199, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L173-L199", "decorators": [], "calls": [ "OpenAI", "client.chat.completions.create", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, 'OpenAI package not installed. Run: pip install openai')", "(True, f'Response: {content[:50]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_completions_endpoint", "qualname": "test_completions_endpoint", "full_name": "examples.test_openai_compatibility.test_completions_endpoint", "kind": "function", "signature": "def test_completions_endpoint(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test /v1/completions endpoint (legacy).", "summary": "Test /v1/completions endpoint (legacy).", "implementation": "Function `test_completions_endpoint` calls `requests.post`, `response.json`, `len`, `data['choices'][0].get`; has 5 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 202, "end_line": 233, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L202-L233", "decorators": [], "calls": [ "requests.post", "response.json", "len", "data['choices'][0].get", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, f'Status code: {response.status_code}')", "(False, \"Missing 'choices' in response\")", "(False, 'Empty choices array')", "(True, f'Response: {text[:50]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_image_chat_http", "qualname": "test_image_chat_http", "full_name": "examples.test_openai_compatibility.test_image_chat_http", "kind": "function", "signature": "def test_image_chat_http(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test multimodal image chat with direct HTTP.", "summary": "Test multimodal image chat with direct HTTP.", "implementation": "Function `test_image_chat_http` calls `create_test_image`, `base64.b64encode(image_bytes).decode`, `base64.b64encode`, `requests.post`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 236, "end_line": 278, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L236-L278", "decorators": [], "calls": [ "create_test_image", "base64.b64encode(image_bytes).decode", "base64.b64encode", "requests.post", "response.json", "str", "Path(image_path).unlink", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, f'Status code: {response.status_code}, Body: {response.text[:100]}')", "(True, f'Response: {content[:50]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_image_chat_openai", "qualname": "test_image_chat_openai", "full_name": "examples.test_openai_compatibility.test_image_chat_openai", "kind": "function", "signature": "def test_image_chat_openai(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test multimodal image chat with OpenAI client.", "summary": "Test multimodal image chat with OpenAI client.", "implementation": "Function `test_image_chat_openai` calls `create_test_image`, `base64.b64encode(image_bytes).decode`, `base64.b64encode`, `OpenAI`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 281, "end_line": 322, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L281-L322", "decorators": [], "calls": [ "create_test_image", "base64.b64encode(image_bytes).decode", "base64.b64encode", "OpenAI", "client.chat.completions.create", "str", "Path(image_path).unlink", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, 'OpenAI package not installed')", "(True, f'Response: {content[:50]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_image_url_http", "qualname": "test_image_url_http", "full_name": "examples.test_openai_compatibility.test_image_url_http", "kind": "function", "signature": "def test_image_url_http(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test image from URL.", "summary": "Test image from URL.", "implementation": "Function `test_image_url_http` calls `requests.post`, `response.json`, `str`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 325, "end_line": 363, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L325-L363", "decorators": [], "calls": [ "requests.post", "response.json", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, f'Status code: {response.status_code}')", "(True, f'Response: {content[:80]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_streaming_chat", "qualname": "test_streaming_chat", "full_name": "examples.test_openai_compatibility.test_streaming_chat", "kind": "function", "signature": "def test_streaming_chat(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test streaming chat completions.", "summary": "Test streaming chat completions.", "implementation": "Function `test_streaming_chat` calls `requests.post`, `response.iter_lines`, `line.decode`, `line.startswith`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 366, "end_line": 405, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L366-L405", "decorators": [], "calls": [ "requests.post", "response.iter_lines", "line.decode", "line.startswith", "chunks.append", "len", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, f'Status code: {response.status_code}')", "(False, 'No streaming chunks received')", "(True, f'Received {len(chunks)} streaming chunks')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "create_test_video", "qualname": "create_test_video", "full_name": "examples.test_openai_compatibility.create_test_video", "kind": "function", "signature": "def create_test_video() -> tuple[str, bytes]", "parameters": [], "return_annotation": "tuple[str, bytes]", "docstring": "Create a simple test video with colored frames.\n\nReturns (path, bytes) of a minimal MP4 video.", "summary": "Create a simple test video with colored frames.", "implementation": "Function `create_test_video` calls `tempfile.NamedTemporaryFile`, `temp_file.close`, `cv2.VideoWriter_fourcc`, `cv2.VideoWriter`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 408, "end_line": 449, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L408-L449", "decorators": [], "calls": [ "tempfile.NamedTemporaryFile", "temp_file.close", "cv2.VideoWriter_fourcc", "cv2.VideoWriter", "np.zeros", "out.write", "out.release", "open", "f.read", "print_warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(temp_path, video_bytes)", "(None, None)" ], "awaits": false, "yields": false }, { "name": "test_video_chat_http", "qualname": "test_video_chat_http", "full_name": "examples.test_openai_compatibility.test_video_chat_http", "kind": "function", "signature": "def test_video_chat_http(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test multimodal video chat with direct HTTP.", "summary": "Test multimodal video chat with direct HTTP.", "implementation": "Function `test_video_chat_http` calls `create_test_video`, `base64.b64encode(video_bytes).decode`, `base64.b64encode`, `requests.post`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 452, "end_line": 498, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L452-L498", "decorators": [], "calls": [ "create_test_video", "base64.b64encode(video_bytes).decode", "base64.b64encode", "requests.post", "response.json", "str", "Path(video_path).unlink", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, 'Could not create test video (OpenCV required)')", "(False, f'Status code: {response.status_code}, Body: {response.text[:100]}')", "(True, f'Response: {content[:80]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_video_chat_openai", "qualname": "test_video_chat_openai", "full_name": "examples.test_openai_compatibility.test_video_chat_openai", "kind": "function", "signature": "def test_video_chat_openai(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test multimodal video chat with OpenAI client.", "summary": "Test multimodal video chat with OpenAI client.", "implementation": "Function `test_video_chat_openai` calls `create_test_video`, `base64.b64encode(video_bytes).decode`, `base64.b64encode`, `OpenAI`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 501, "end_line": 546, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L501-L546", "decorators": [], "calls": [ "create_test_video", "base64.b64encode(video_bytes).decode", "base64.b64encode", "OpenAI", "client.chat.completions.create", "str", "Path(video_path).unlink", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, 'OpenAI package not installed')", "(False, 'Could not create test video (OpenCV required)')", "(True, f'Response: {content[:80]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "test_video_url_http", "qualname": "test_video_url_http", "full_name": "examples.test_openai_compatibility.test_video_url_http", "kind": "function", "signature": "def test_video_url_http(server_url: str) -> tuple[bool, str]", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "Test video from URL.", "summary": "Test video from URL.", "implementation": "Function `test_video_url_http` calls `requests.post`, `response.json`, `str`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 549, "end_line": 587, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L549-L587", "decorators": [], "calls": [ "requests.post", "response.json", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, f'Status code: {response.status_code}')", "(True, f'Response: {content[:80]}...')", "(False, str(e))" ], "awaits": false, "yields": false }, { "name": "run_all_tests", "qualname": "run_all_tests", "full_name": "examples.test_openai_compatibility.run_all_tests", "kind": "function", "signature": "def run_all_tests(server_url: str, test_image: bool=True, test_video: bool=True)", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "test_image", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Optional positional or keyword input; defaults to `True`." }, { "name": "test_video", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Optional positional or keyword input; defaults to `True`." } ], "return_annotation": "not annotated", "docstring": "Run all compatibility tests.", "summary": "Run all compatibility tests.", "implementation": "Function `run_all_tests` calls `print_header`, `print`, `test_health_endpoint`, `print_test`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 590, "end_line": 684, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L590-L684", "decorators": [], "calls": [ "print_header", "print", "test_health_endpoint", "print_test", "record", "test_models_endpoint", "test_chat_completions_http", "test_chat_completions_openai", "test_completions_endpoint", "test_streaming_chat", "test_image_chat_http", "test_image_chat_openai", "test_image_url_http", "test_video_chat_http", "test_video_chat_openai", "test_video_url_http" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0", "1" ], "awaits": false, "yields": false }, { "name": "record", "qualname": "run_all_tests.record", "full_name": "examples.test_openai_compatibility.run_all_tests.record", "kind": "nested function", "signature": "def record(passed: bool)", "parameters": [ { "name": "passed", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `run_all_tests.record` contains no state mutation, call, raise, return, await, or yield.", "implementation": "Nested Function `run_all_tests.record` contains no state mutation, call, raise, return, await, or yield.", "documented": false, "public": false, "addressable": false, "line": 594, "end_line": 598, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L594-L598", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.test_openai_compatibility.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `run_all_tests(server_url=args.server_url, test_image=not args.no_image, test_video=not args.no_video)`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `run_all_tests(server_url=args.server_url, test_image=not args.no_image, test_video=not args.no_video)`.", "documented": false, "public": true, "addressable": true, "line": 687, "end_line": 735, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_openai_compatibility.py#L687-L735", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "test_health_endpoint", "sys.exit", "run_all_tests" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "run_all_tests(server_url=args.server_url, test_image=not args.no_image, test_video=not args.no_video)" ], "awaits": false, "yields": false } ] }, { "name": "examples.test_video", "path": "examples/test_video.py", "page_path": "reference/source/examples/test_video.md", "docstring": "Test script for VLM video support in vllm-mlx.\n\nThis script tests video understanding capabilities by:\n1. Downloading a sample video (or using a local one)\n2. Loading a VLM model that supports video\n3. Running inference on the video\n\nUsage:\n python examples/test_video.py\n python examples/test_video.py --video /path/to/video.mp4\n python examples/test_video.py --model mlx-community/Qwen3-VL-8B-Instruct-4bit", "summary": "Test script for VLM video support in vllm-mlx.", "line_count": 349, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L1-L349", "members": [ "logger", "download_sample_video", "create_test_video", "get_video_info", "test_frame_extraction", "test_video_generation", "test_video_url", "main" ], "symbols": [ { "name": "download_sample_video", "qualname": "download_sample_video", "full_name": "examples.test_video.download_sample_video", "kind": "function", "signature": "def download_sample_video() -> str", "parameters": [], "return_annotation": "str", "docstring": "Download a sample video for testing.", "summary": "Download a sample video for testing.", "implementation": "Function `download_sample_video` calls `logger.info`, `requests.get`, `response.raise_for_status`, `tempfile.NamedTemporaryFile`; returns `temp_file.name`.", "documented": true, "public": true, "addressable": true, "line": 27, "end_line": 56, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L27-L56", "decorators": [], "calls": [ "logger.info", "requests.get", "response.raise_for_status", "tempfile.NamedTemporaryFile", "response.iter_content", "temp_file.write", "temp_file.close", "logger.error", "sys.exit" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "temp_file.name" ], "awaits": false, "yields": false }, { "name": "create_test_video", "qualname": "create_test_video", "full_name": "examples.test_video.create_test_video", "kind": "function", "signature": "def create_test_video() -> str", "parameters": [], "return_annotation": "str", "docstring": "Create a simple test video using OpenCV if download fails.", "summary": "Create a simple test video using OpenCV if download fails.", "implementation": "Function `create_test_video` calls `logger.error`, `sys.exit`, `logger.info`, `tempfile.NamedTemporaryFile`; returns `temp_file.name`.", "documented": true, "public": true, "addressable": true, "line": 59, "end_line": 105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L59-L105", "decorators": [], "calls": [ "logger.error", "sys.exit", "logger.info", "tempfile.NamedTemporaryFile", "temp_file.close", "cv2.VideoWriter_fourcc", "cv2.VideoWriter", "range", "np.zeros", "len", "cv2.putText", "out.write", "out.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "temp_file.name" ], "awaits": false, "yields": false }, { "name": "get_video_info", "qualname": "get_video_info", "full_name": "examples.test_video.get_video_info", "kind": "function", "signature": "def get_video_info(video_path: str) -> dict", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Get information about a video file.", "summary": "Get information about a video file.", "implementation": "Function `get_video_info` calls `cv2.VideoCapture`, `cap.isOpened`, `int`, `cap.get`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 108, "end_line": 126, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L108-L126", "decorators": [], "calls": [ "cv2.VideoCapture", "cap.isOpened", "int", "cap.get", "cap.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'error': 'Cannot open video'}", "info" ], "awaits": false, "yields": false }, { "name": "test_frame_extraction", "qualname": "test_frame_extraction", "full_name": "examples.test_video.test_frame_extraction", "kind": "function", "signature": "def test_frame_extraction(video_path: str)", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Test video frame extraction.", "summary": "Test video frame extraction.", "implementation": "Function `test_frame_extraction` calls `logger.info`, `get_video_info`, `time.time`, `extract_video_frames_smart`.", "documented": true, "public": true, "addressable": true, "line": 129, "end_line": 145, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L129-L145", "decorators": [], "calls": [ "logger.info", "get_video_info", "time.time", "extract_video_frames_smart", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "test_video_generation", "qualname": "test_video_generation", "full_name": "examples.test_video.test_video_generation", "kind": "function", "signature": "def test_video_generation(video_path: str, model_name: str)", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Test video understanding with VLM.", "summary": "Test video understanding with VLM.", "implementation": "Function `test_video_generation` calls `logger.info`, `time.time`, `MLXVisionLanguageModel`, `model.load`; returns `model`.", "documented": true, "public": true, "addressable": true, "line": 148, "end_line": 219, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L148-L219", "decorators": [], "calls": [ "logger.info", "time.time", "MLXVisionLanguageModel", "model.load", "model.describe_video", "model.generate", "model.chat" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "model" ], "awaits": false, "yields": false }, { "name": "test_video_url", "qualname": "test_video_url", "full_name": "examples.test_video.test_video_url", "kind": "function", "signature": "def test_video_url(model, video_url: str)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Test video from URL.", "summary": "Test video from URL.", "implementation": "Function `test_video_url` calls `logger.info`, `time.time`, `model.chat`, `model.generate`; returns `True`.", "documented": true, "public": true, "addressable": true, "line": 222, "end_line": 266, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L222-L266", "decorators": [], "calls": [ "logger.info", "time.time", "model.chat", "model.generate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.test_video.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `Path(video_path).exists`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `Path(video_path).exists`.", "documented": false, "public": true, "addressable": true, "line": 269, "end_line": 345, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/test_video.py#L269-L345", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "Path(video_path).exists", "Path", "logger.error", "sys.exit", "create_test_video", "download_sample_video", "logger.info", "test_frame_extraction", "test_video_generation", "MLXVisionLanguageModel", "model.load", "test_video_url" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "examples.tts_example", "path": "examples/tts_example.py", "page_path": "reference/source/examples/tts_example.md", "docstring": "TTS Example - Text to Speech with vllm-mlx\n\nUsage:\n python examples/tts_example.py \"Hello, how are you?\"\n python examples/tts_example.py \"Welcome!\" --voice am_michael\n python examples/tts_example.py \"Hola, como estas?\" --lang es\n python examples/tts_example.py --list-voices\n python examples/tts_example.py --list-languages", "summary": "TTS Example - Text to Speech with vllm-mlx Usage: python examples/tts_example.py \"Hello, how are you?\" python examples/tts_example.py \"Welcome!\" --voice am_michael python examples/tts_example.py \"Hola, como estas?\" --lang es python examples/tts_example.py --list-voices python examples/tts_example.py --list-languages", "line_count": 138, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_example.py#L1-L138", "members": [ "LANGUAGES", "LANG_ALIASES", "main" ], "symbols": [ { "name": "main", "qualname": "main", "full_name": "examples.tts_example.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 46, "end_line": 134, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_example.py#L46-L134", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "LANGUAGES.items", "sorted", "LANG_ALIASES.items", "args.lang.lower", "LANG_ALIASES.get", "LANGUAGES.get", "TTSEngine", "engine.load", "engine.get_voices", "len", "engine.generate", "engine.save", "os.system" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "examples.tts_multilingual", "path": "examples/tts_multilingual.py", "page_path": "reference/source/examples/tts_multilingual.md", "docstring": "Multilingual TTS Example - Text to Speech with multiple models and languages\n\nSupported Models:\n - Kokoro: Fast, 82M params, 8 languages (en, es, fr, ja, zh, hi, it, pt)\n - Chatterbox: Expressive, voice cloning, 15+ languages\n - VibeVoice: Realtime, low latency, English\n - VoxCPM: High quality, Chinese/English\n - OuteTTS: Voice cloning, en/zh/ja/ko\n - Spark: Voice cloning, en/zh\n\nUsage:\n python examples/tts_multilingual.py \"Hello world\"\n python examples/tts_multilingual.py \"Hola mundo\" --lang es\n python examples/tts_multilingual.py \"Bonjour le monde\" --lang fr --model kokoro\n python examples/tts_multilingual.py --list-models\n python examples/tts_multilingual.py --list-languages", "summary": "Multilingual TTS Example - Text to Speech with multiple models and languages Supported Models: - Kokoro: Fast, 82M params, 8 languages (en, es, fr, ja, zh, hi, it, pt) - Chatterbox: Expressive, voice cloning, 15+ languages - VibeVoice: Realtime, low latency, English - VoxCPM: High quality, Chinese/English - OuteTTS: Voice cloning, en/zh/ja/ko - Spark: Voice cloning, en/zh Usage: python examples/tts_multilingual.py \"Hello world\" python examples/tts_multilingual.py \"Hola mundo\" --lang es python examples/tts_multilingual.py \"Bonjour le monde\" --lang fr --model kokoro python examples/tts_multilingual.py --list-models python examples/tts_multilingual.py --list-languages", "line_count": 340, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_multilingual.py#L1-L340", "members": [ "MODELS", "LANGUAGES", "get_best_model_for_language", "list_models", "list_languages", "generate_speech", "main" ], "symbols": [ { "name": "get_best_model_for_language", "qualname": "get_best_model_for_language", "full_name": "examples.tts_multilingual.get_best_model_for_language", "kind": "function", "signature": "def get_best_model_for_language(lang: str) -> str", "parameters": [ { "name": "lang", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Get the best model for a given language.", "summary": "Get the best model for a given language.", "implementation": "Function `get_best_model_for_language` calls `lang.lower`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 109, "end_line": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_multilingual.py#L109-L123", "decorators": [], "calls": [ "lang.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'kokoro'", "'voxcpm'", "'chatterbox'" ], "awaits": false, "yields": false }, { "name": "list_models", "qualname": "list_models", "full_name": "examples.tts_multilingual.list_models", "kind": "function", "signature": "def list_models()", "parameters": [], "return_annotation": "not annotated", "docstring": "Print available models.", "summary": "Print available models.", "implementation": "Function `list_models` calls `print`, `MODELS.items`, `', '.join`, `len`.", "documented": true, "public": true, "addressable": true, "line": 126, "end_line": 138, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_multilingual.py#L126-L138", "decorators": [], "calls": [ "print", "MODELS.items", "', '.join", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "list_languages", "qualname": "list_languages", "full_name": "examples.tts_multilingual.list_languages", "kind": "function", "signature": "def list_languages()", "parameters": [], "return_annotation": "not annotated", "docstring": "Print available languages and best models.", "summary": "Print available languages and best models.", "implementation": "Function `list_languages` calls `print`, `sorted`, `LANGUAGES.items`, `get_best_model_for_language`.", "documented": true, "public": true, "addressable": true, "line": 141, "end_line": 152, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_multilingual.py#L141-L152", "decorators": [], "calls": [ "print", "sorted", "LANGUAGES.items", "get_best_model_for_language", "MODELS.items", "', '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "generate_speech", "qualname": "generate_speech", "full_name": "examples.tts_multilingual.generate_speech", "kind": "function", "signature": "def generate_speech(text: str, model_name: str, lang: str, voice: str, speed: float, output: str)", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "lang", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "voice", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "speed", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Generate speech using the specified model.", "summary": "Generate speech using the specified model.", "implementation": "Function `generate_speech` calls `print`, `LANGUAGES.get(lang, {}).get`, `LANGUAGES.get`, `time.time`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 155, "end_line": 244, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_multilingual.py#L155-L244", "decorators": [], "calls": [ "print", "LANGUAGES.get(lang, {}).get", "LANGUAGES.get", "time.time", "load_model", "lang_info.get", "model.generate", "hasattr", "np.array", "audio_data.tolist", "audio_chunks.append", "len", "np.concatenate", "(full_audio * 32767).astype", "wave.open", "wf.setnchannels", "wf.setsampwidth", "wf.setframerate", "wf.writeframes", "audio_int16.tobytes" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "output" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.tts_multilingual.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`; returns `None`.", "documented": false, "public": true, "addressable": true, "line": 247, "end_line": 336, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/tts_multilingual.py#L247-L336", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "list_models", "list_languages", "parser.print_help", "get_best_model_for_language", "', '.join", "MODELS.keys", "generate_speech", "os.system" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "examples.video_benchmark", "path": "examples/video_benchmark.py", "page_path": "reference/source/examples/video_benchmark.md", "docstring": "Video Benchmark Script for vllm-mlx\n\nTests Vision-Language Models with video at different configurations\n(FPS, frame count, resolution) and measures performance metrics.\n\nUsage:\n # Direct API benchmark (no server needed):\n python examples/video_benchmark.py --model mlx-community/Qwen3-VL-4B-Instruct-3bit\n\n # With video URL:\n python examples/video_benchmark.py --video-url https://example.com/video.mp4\n\n # Quick test:\n python examples/video_benchmark.py --quick", "summary": "Video Benchmark Script for vllm-mlx Tests Vision-Language Models with video at different configurations (FPS, frame count, resolution) and measures performance metrics.", "line_count": 563, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L1-L563", "members": [ "logger", "SAMPLE_VIDEOS", "VideoBenchmarkResult", "create_test_video", "download_video", "get_video_info", "run_video_benchmark", "run_benchmark", "print_results", "save_results", "main" ], "symbols": [ { "name": "VideoBenchmarkResult", "qualname": "VideoBenchmarkResult", "full_name": "examples.video_benchmark.VideoBenchmarkResult", "kind": "class", "signature": "class VideoBenchmarkResult", "parameters": [ { "name": "config_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "fps", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "max_frames", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "frames_extracted", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "video_duration", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "time_seconds", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "completion_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens_per_second", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "response_preview", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "VideoBenchmarkResult", "docstring": "Result from a single video benchmark run.", "summary": "Result from a single video benchmark run.", "implementation": "Class `VideoBenchmarkResult` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 80, "end_line": 91, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L80-L91", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "create_test_video", "qualname": "create_test_video", "full_name": "examples.video_benchmark.create_test_video", "kind": "function", "signature": "def create_test_video(duration: float=5.0, fps: float=30.0, width: int=640, height: int=480) -> str", "parameters": [ { "name": "duration", "kind": "positional or keyword", "annotation": "float", "default": "5.0", "required": false, "description": "Video duration in seconds" }, { "name": "fps", "kind": "positional or keyword", "annotation": "float", "default": "30.0", "required": false, "description": "Frames per second" }, { "name": "width", "kind": "positional or keyword", "annotation": "int", "default": "640", "required": false, "description": "Video width" }, { "name": "height", "kind": "positional or keyword", "annotation": "int", "default": "480", "required": false, "description": "Video height" } ], "return_annotation": "str", "docstring": "Create a synthetic test video with colored frames and text.\n\nArgs:\n duration: Video duration in seconds\n fps: Frames per second\n width: Video width\n height: Video height\n\nReturns:\n Path to created video file", "summary": "Create a synthetic test video with colored frames and text.", "implementation": "Function `create_test_video` calls `tempfile.NamedTemporaryFile`, `temp_file.close`, `cv2.VideoWriter_fourcc`, `cv2.VideoWriter`; returns `temp_file.name`.", "documented": true, "public": true, "addressable": true, "line": 94, "end_line": 175, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L94-L175", "decorators": [], "calls": [ "tempfile.NamedTemporaryFile", "temp_file.close", "cv2.VideoWriter_fourcc", "cv2.VideoWriter", "int", "len", "range", "np.zeros", "min", "cv2.putText", "out.write", "out.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "temp_file.name" ], "awaits": false, "yields": false }, { "name": "download_video", "qualname": "download_video", "full_name": "examples.video_benchmark.download_video", "kind": "function", "signature": "def download_video(url: str, timeout: int=120) -> str", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "120", "required": false, "description": "Optional positional or keyword input; defaults to `120`." } ], "return_annotation": "str", "docstring": "Download video from URL.", "summary": "Download video from URL.", "implementation": "Function `download_video` calls `logger.info`, `requests.get`, `response.raise_for_status`, `tempfile.NamedTemporaryFile`; returns `temp_file.name`.", "documented": true, "public": true, "addressable": true, "line": 178, "end_line": 198, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L178-L198", "decorators": [], "calls": [ "logger.info", "requests.get", "response.raise_for_status", "tempfile.NamedTemporaryFile", "response.iter_content", "temp_file.write", "temp_file.close", "Path(temp_file.name).stat", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "temp_file.name" ], "awaits": false, "yields": false }, { "name": "get_video_info", "qualname": "get_video_info", "full_name": "examples.video_benchmark.get_video_info", "kind": "function", "signature": "def get_video_info(video_path: str) -> dict", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Get information about a video file.", "summary": "Get information about a video file.", "implementation": "Function `get_video_info` calls `cv2.VideoCapture`, `cap.isOpened`, `int`, `cap.get`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 201, "end_line": 217, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L201-L217", "decorators": [], "calls": [ "cv2.VideoCapture", "cap.isOpened", "int", "cap.get", "cap.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'error': 'Cannot open video'}", "info" ], "awaits": false, "yields": false }, { "name": "run_video_benchmark", "qualname": "run_video_benchmark", "full_name": "examples.video_benchmark.run_video_benchmark", "kind": "function", "signature": "def run_video_benchmark(model, video_path: str, fps: float, max_frames: int, config_name: str, warmup: bool=False) -> VideoBenchmarkResult", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "fps", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_frames", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "warmup", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "VideoBenchmarkResult", "docstring": "Run a single video benchmark configuration.", "summary": "Run a single video benchmark configuration.", "implementation": "Function `run_video_benchmark` calls `get_video_info`, `print`, `time.perf_counter`, `model.generate`; returns `VideoBenchmarkResult(config_name=config_name, fps=fps, max_frames=max_frames, frames_extracted=frames_extracted, video_…`.", "documented": true, "public": true, "addressable": true, "line": 220, "end_line": 271, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L220-L271", "decorators": [], "calls": [ "get_video_info", "print", "time.perf_counter", "model.generate", "int", "min", "VideoBenchmarkResult", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "VideoBenchmarkResult(config_name=config_name, fps=fps, max_frames=max_frames, frames_extracted=frames_extracted, video_…" ], "awaits": false, "yields": false }, { "name": "run_benchmark", "qualname": "run_benchmark", "full_name": "examples.video_benchmark.run_benchmark", "kind": "function", "signature": "def run_benchmark(model_name: str, video_path: str=None, video_url: str=None, video_duration: float=10.0, warmup_runs: int=1, quick: bool=False) -> list[VideoBenchmarkResult]", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "VLM model to use" }, { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Local video file path" }, { "name": "video_url", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "URL to download video from" }, { "name": "video_duration", "kind": "positional or keyword", "annotation": "float", "default": "10.0", "required": false, "description": "Duration for synthetic video" }, { "name": "warmup_runs", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Number of warmup runs" }, { "name": "quick", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Run quick benchmark with fewer configs" } ], "return_annotation": "list[VideoBenchmarkResult]", "docstring": "Run full video benchmark across multiple configurations.\n\nArgs:\n model_name: VLM model to use\n video_path: Local video file path\n video_url: URL to download video from\n video_duration: Duration for synthetic video\n warmup_runs: Number of warmup runs\n quick: Run quick benchmark with fewer configs\n\nReturns:\n List of VideoBenchmarkResult objects", "summary": "Run full video benchmark across multiple configurations.", "implementation": "Function `run_benchmark` calls `print`, `time.time`, `MLXVisionLanguageModel`, `model.load`; returns `results`.", "documented": true, "public": true, "addressable": true, "line": 274, "end_line": 374, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L274-L374", "decorators": [], "calls": [ "print", "time.time", "MLXVisionLanguageModel", "model.load", "Path(video_path).exists", "Path", "download_video", "create_test_video", "get_video_info", "range", "run_video_benchmark", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "print_results", "qualname": "print_results", "full_name": "examples.video_benchmark.print_results", "kind": "function", "signature": "def print_results(results: list[VideoBenchmarkResult])", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[VideoBenchmarkResult]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print benchmark results in a nice table.", "summary": "Print benchmark results in a nice table.", "implementation": "Function `print_results` calls `print`, `sorted`, `table_data.append`, `tabulate`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 377, "end_line": 445, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L377-L445", "decorators": [], "calls": [ "print", "sorted", "table_data.append", "tabulate", "sum", "min", "max", "frame_groups[key].append", "frame_groups.keys", "len", "analysis_data.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "save_results", "qualname": "save_results", "full_name": "examples.video_benchmark.save_results", "kind": "function", "signature": "def save_results(results: list[VideoBenchmarkResult], output_path: str, model_name: str)", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[VideoBenchmarkResult]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "output_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Save benchmark results to JSON file.", "summary": "Save benchmark results to JSON file.", "implementation": "Function `save_results` calls `time.strftime`, `open`, `json.dump`, `print`.", "documented": true, "public": true, "addressable": true, "line": 448, "end_line": 474, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L448-L474", "decorators": [], "calls": [ "time.strftime", "open", "json.dump", "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "examples.video_benchmark.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `run_benchmark`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `run_benchmark`.", "documented": false, "public": true, "addressable": true, "line": 477, "end_line": 559, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/examples/video_benchmark.py#L477-L559", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "run_benchmark", "print_results", "save_results" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "scripts.add_mtp_weights", "path": "scripts/add_mtp_weights.py", "page_path": "reference/source/scripts/add_mtp_weights.md", "docstring": "Add MTP (Multi-Token Prediction) weights to an existing MLX Qwen3-Next model.\n\nThis script:\n1. Downloads the MTP shard from the original BF16 HuggingFace model\n2. Extracts MTP weights (mtp.* keys)\n3. Quantizes them to match the existing MLX model's quantization\n4. Adds them to the MLX model's safetensors files\n5. Updates config.json with num_nextn_predict_layers=1\n\nUsage:\n ``python add_mtp_weights.py [--mlx-model-path PATH] [--source-model MODEL]``\n\nRequirements:\n pip install mlx", "summary": "Add MTP (Multi-Token Prediction) weights to an existing MLX Qwen3-Next model.", "line_count": 341, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L1-L341", "members": [ "DEFAULT_MLX_MODEL", "DEFAULT_SOURCE_MODEL", "MTP_SHARD_NAME", "MTP_SHARD_URL", "find_snapshot_dir", "download_mtp_shard", "extract_and_quantize_mtp_weights", "update_model_index", "update_config", "main" ], "symbols": [ { "name": "find_snapshot_dir", "qualname": "find_snapshot_dir", "full_name": "scripts.add_mtp_weights.find_snapshot_dir", "kind": "function", "signature": "def find_snapshot_dir(model_path: str) -> Path", "parameters": [ { "name": "model_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Path", "docstring": "Find the latest snapshot directory in HF cache structure.", "summary": "Find the latest snapshot directory in HF cache structure.", "implementation": "Function `find_snapshot_dir` calls `Path`, `snapshots_dir.exists`, `(Path(model_path) / 'config.json').exists`, `FileNotFoundError`; can raise `FileNotFoundError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 38, "end_line": 51, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L38-L51", "decorators": [], "calls": [ "Path", "snapshots_dir.exists", "(Path(model_path) / 'config.json').exists", "FileNotFoundError", "sorted", "snapshots_dir.iterdir" ], "state_reads": [], "state_writes": [], "raises": [ "FileNotFoundError" ], "return_expressions": [ "Path(model_path)", "snapshots[-1]" ], "awaits": false, "yields": false }, { "name": "download_mtp_shard", "qualname": "download_mtp_shard", "full_name": "scripts.add_mtp_weights.download_mtp_shard", "kind": "function", "signature": "def download_mtp_shard(dest_path: Path, source_model: str) -> Path", "parameters": [ { "name": "dest_path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "source_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Path", "docstring": "Download the MTP shard using curl with resume support.", "summary": "Download the MTP shard using curl with resume support.", "implementation": "Function `download_mtp_shard` calls `shard_path.exists`, `print`, `shard_path.stat`, `subprocess.run`; can raise `RuntimeError`; returns `shard_path`.", "documented": true, "public": true, "addressable": true, "line": 54, "end_line": 82, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L54-L82", "decorators": [], "calls": [ "shard_path.exists", "print", "shard_path.stat", "subprocess.run", "str", "RuntimeError" ], "state_reads": [], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "shard_path" ], "awaits": false, "yields": false }, { "name": "extract_and_quantize_mtp_weights", "qualname": "extract_and_quantize_mtp_weights", "full_name": "scripts.add_mtp_weights.extract_and_quantize_mtp_weights", "kind": "function", "signature": "def extract_and_quantize_mtp_weights(shard_path: Path, snapshot_dir: Path, quantization_bits: int=6)", "parameters": [ { "name": "shard_path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "snapshot_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "quantization_bits", "kind": "positional or keyword", "annotation": "int", "default": "6", "required": false, "description": "Optional positional or keyword input; defaults to `6`." } ], "return_annotation": "not annotated", "docstring": "Extract MTP weights, quantize, and save to MLX model directory.", "summary": "Extract MTP weights, quantize, and save to MLX model directory.", "implementation": "Function `extract_and_quantize_mtp_weights` calls `mx.set_default_device`, `print`, `mx.load`, `str`; returns `(mtp_output_file, list(quantized_weights.keys()))`.", "documented": true, "public": true, "addressable": true, "line": 85, "end_line": 195, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L85-L195", "decorators": [], "calls": [ "mx.set_default_device", "print", "mx.load", "str", "all_weights.items", "k.startswith", "len", "open", "json.load", "config.get", "quant_config.get", "range", "all", "mx.stack", "mtp_weights.pop", "mx.eval", "quantized_weights.update", "_quantize_one", "list", "mtp_weights.keys", "mx.save_safetensors", "sum", "quantized_weights.values", "quantized_weights.keys" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(mtp_output_file, list(quantized_weights.keys()))" ], "awaits": false, "yields": false }, { "name": "_quantize_one", "qualname": "extract_and_quantize_mtp_weights._quantize_one", "full_name": "scripts.add_mtp_weights.extract_and_quantize_mtp_weights._quantize_one", "kind": "nested function", "signature": "def _quantize_one(key, weight)", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weight", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Quantize a single weight, apply norm adjustment, return dict entries.", "summary": "Quantize a single weight, apply norm adjustment, return dict entries.", "implementation": "Nested Function `extract_and_quantize_mtp_weights._quantize_one` calls `any`, `key.endswith`, `mx.eval`, `print`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 137, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L137-L160", "decorators": [], "calls": [ "any", "key.endswith", "mx.eval", "print", "mx.quantize", "key.replace" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{key: weight}", "{key: q_w, key.replace('.weight', '.scales'): q_s, key.replace('.weight', '.biases'): q_b}" ], "awaits": false, "yields": false }, { "name": "update_model_index", "qualname": "update_model_index", "full_name": "scripts.add_mtp_weights.update_model_index", "kind": "function", "signature": "def update_model_index(snapshot_dir: Path, mtp_keys: list)", "parameters": [ { "name": "snapshot_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "mtp_keys", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Update model.safetensors.index.json to include MTP weight keys.", "summary": "Update model.safetensors.index.json to include MTP weight keys.", "implementation": "Function `update_model_index` calls `index_path.exists`, `print`, `open`, `json.load`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 198, "end_line": 221, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L198-L221", "decorators": [], "calls": [ "index_path.exists", "print", "open", "json.load", "index.get", "json.dump", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "update_config", "qualname": "update_config", "full_name": "scripts.add_mtp_weights.update_config", "kind": "function", "signature": "def update_config(snapshot_dir: Path)", "parameters": [ { "name": "snapshot_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Update config.json to enable MTP.", "summary": "Update config.json to enable MTP.", "implementation": "Function `update_config` calls `open`, `json.load`, `json.dump`, `print`.", "documented": true, "public": true, "addressable": true, "line": 224, "end_line": 236, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L224-L236", "decorators": [], "calls": [ "open", "json.load", "json.dump", "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "scripts.add_mtp_weights.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "documented": false, "public": true, "addressable": true, "line": 239, "end_line": 337, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights.py#L239-L337", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "find_snapshot_dir", "config_path.exists", "sys.exit", "open", "json.load", "config.get", "index_path.exists", "index.get", "k.startswith", "len", "Path", "tempfile.mkdtemp", "download_mtp_shard", "shard_path.exists", "extract_and_quantize_mtp_weights", "update_model_index", "update_config" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "scripts.add_mtp_weights_qwen35", "path": "scripts/add_mtp_weights_qwen35.py", "page_path": "reference/source/scripts/add_mtp_weights_qwen35.md", "docstring": "Add MTP (Multi-Token Prediction) weights to an existing MLX Qwen3.5 model.\n\nThis script:\n1. Fetches the safetensors index from the original BF16 HuggingFace model\n2. Identifies shards containing MTP weights (mtp.* keys)\n3. Downloads only those shards via curl -C -\n4. Extracts MTP weights\n5. For MoE models: stacks expert weights (256×) into switch_mlp format\n6. Applies norm shift (HF weight → MLX weight+1.0) for RMSNorm keys\n7. Quantizes to match the MLX model's quantization scheme\n8. Saves as mtp/weights.safetensors (subdirectory avoids mlx_vlm glob)\n\nSupports both:\n- MoE models (Qwen3.5-122B-A10B, 35B-A3B): 256 experts, sparse MTP attention\n- Dense models (Qwen3.5-27B): full MTP with k/v projections and norms\n\nUsage:\n python add_mtp_weights_qwen35.py --mlx-model-path PATH --source-model MODEL\n\nRequirements:\n pip install mlx", "summary": "Add MTP (Multi-Token Prediction) weights to an existing MLX Qwen3.5 model.", "line_count": 470, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L1-L470", "members": [ "MODEL_CONFIGS", "find_snapshot_dir", "fetch_shard_index", "identify_mtp_shards", "download_shards", "extract_and_quantize_mtp_weights", "update_model_index", "update_config", "main" ], "symbols": [ { "name": "find_snapshot_dir", "qualname": "find_snapshot_dir", "full_name": "scripts.add_mtp_weights_qwen35.find_snapshot_dir", "kind": "function", "signature": "def find_snapshot_dir(model_path: str) -> Path", "parameters": [ { "name": "model_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Path", "docstring": "Find the latest snapshot directory in HF cache structure.", "summary": "Find the latest snapshot directory in HF cache structure.", "implementation": "Function `find_snapshot_dir` calls `Path`, `snapshots_dir.exists`, `(Path(model_path) / 'config.json').exists`, `FileNotFoundError`; can raise `FileNotFoundError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 53, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L53-L63", "decorators": [], "calls": [ "Path", "snapshots_dir.exists", "(Path(model_path) / 'config.json').exists", "FileNotFoundError", "sorted", "snapshots_dir.iterdir" ], "state_reads": [], "state_writes": [], "raises": [ "FileNotFoundError" ], "return_expressions": [ "Path(model_path)", "snapshots[-1]" ], "awaits": false, "yields": false }, { "name": "fetch_shard_index", "qualname": "fetch_shard_index", "full_name": "scripts.add_mtp_weights_qwen35.fetch_shard_index", "kind": "function", "signature": "def fetch_shard_index(source_model: str, download_dir: Path) -> dict", "parameters": [ { "name": "source_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "download_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Fetch model.safetensors.index.json from HuggingFace.", "summary": "Fetch model.safetensors.index.json from HuggingFace.", "implementation": "Function `fetch_shard_index` calls `print`, `subprocess.run`, `str`, `RuntimeError`; can raise `RuntimeError`; returns `json.load(f)`.", "documented": true, "public": true, "addressable": true, "line": 66, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L66-L80", "decorators": [], "calls": [ "print", "subprocess.run", "str", "RuntimeError", "open", "json.load" ], "state_reads": [], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "json.load(f)" ], "awaits": false, "yields": false }, { "name": "identify_mtp_shards", "qualname": "identify_mtp_shards", "full_name": "scripts.add_mtp_weights_qwen35.identify_mtp_shards", "kind": "function", "signature": "def identify_mtp_shards(index: dict) -> tuple[dict[str, str], set[str]]", "parameters": [ { "name": "index", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[dict[str, str], set[str]]", "docstring": "Identify which shards contain MTP weights.\n\nReturns:\n Tuple of (mtp_key_to_shard mapping, set of shard filenames to download)", "summary": "Identify which shards contain MTP weights.", "implementation": "Function `identify_mtp_shards` calls `index.get`, `set`, `weight_map.items`, `key.startswith`; returns `(mtp_keys, shards_needed)`.", "documented": true, "public": true, "addressable": true, "line": 83, "end_line": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L83-L98", "decorators": [], "calls": [ "index.get", "set", "weight_map.items", "key.startswith", "shards_needed.add" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(mtp_keys, shards_needed)" ], "awaits": false, "yields": false }, { "name": "download_shards", "qualname": "download_shards", "full_name": "scripts.add_mtp_weights_qwen35.download_shards", "kind": "function", "signature": "def download_shards(shards: set[str], source_model: str, download_dir: Path) -> dict[str, Path]", "parameters": [ { "name": "shards", "kind": "positional or keyword", "annotation": "set[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "source_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "download_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Path]", "docstring": "Download required shards using curl with resume support.", "summary": "Download required shards using curl with resume support.", "implementation": "Function `download_shards` calls `sorted`, `shard_path.exists`, `shard_path.stat`, `print`; can raise `RuntimeError`; returns `shard_paths`.", "documented": true, "public": true, "addressable": true, "line": 101, "end_line": 130, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L101-L130", "decorators": [], "calls": [ "sorted", "shard_path.exists", "shard_path.stat", "print", "subprocess.run", "str", "RuntimeError" ], "state_reads": [], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "shard_paths" ], "awaits": false, "yields": false }, { "name": "extract_and_quantize_mtp_weights", "qualname": "extract_and_quantize_mtp_weights", "full_name": "scripts.add_mtp_weights_qwen35.extract_and_quantize_mtp_weights", "kind": "function", "signature": "def extract_and_quantize_mtp_weights(mtp_keys: dict[str, str], shard_paths: dict[str, Path], snapshot_dir: Path, is_moe: bool, num_experts: int, no_quantize: bool=False)", "parameters": [ { "name": "mtp_keys", "kind": "positional or keyword", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "shard_paths", "kind": "positional or keyword", "annotation": "dict[str, Path]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "snapshot_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "is_moe", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_experts", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "no_quantize", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "not annotated", "docstring": "Extract MTP weights from BF16 shards, optionally quantize, and save.", "summary": "Extract MTP weights from BF16 shards, optionally quantize, and save.", "implementation": "Function `extract_and_quantize_mtp_weights` calls `mx.set_default_device`, `open`, `json.load`, `config.get`; returns `(mtp_output_file, list(quantized_weights.keys()))`.", "documented": true, "public": true, "addressable": true, "line": 133, "end_line": 272, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L133-L272", "decorators": [], "calls": [ "mx.set_default_device", "open", "json.load", "config.get", "text_config.get", "quant_config.get", "print", "mtp_keys.items", "shard_to_keys.setdefault(shard, []).append", "shard_to_keys.setdefault", "len", "sorted", "shard_to_keys.items", "mx.load", "str", "range", "all", "mx.stack", "all_mtp_weights.pop", "mx.eval", "quantized_weights.update", "_quantize_one", "sum", "all_mtp_weights.keys", "mtp_output_dir.mkdir", "mx.save_safetensors", "quantized_weights.values", "list", "quantized_weights.keys" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(mtp_output_file, list(quantized_weights.keys()))" ], "awaits": false, "yields": false }, { "name": "_quantize_one", "qualname": "extract_and_quantize_mtp_weights._quantize_one", "full_name": "scripts.add_mtp_weights_qwen35.extract_and_quantize_mtp_weights._quantize_one", "kind": "nested function", "signature": "def _quantize_one(key: str, weight: mx.array) -> dict[str, mx.array]", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weight", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, mx.array]", "docstring": "Quantize a single weight, apply norm adjustment.", "summary": "Quantize a single weight, apply norm adjustment.", "implementation": "Nested Function `extract_and_quantize_mtp_weights._quantize_one` calls `any`, `key.endswith`, `mx.eval`, `print`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 204, "end_line": 229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L204-L229", "decorators": [], "calls": [ "any", "key.endswith", "mx.eval", "print", "mx.quantize", "key.replace" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{key: weight}", "{key: q_w, key.replace('.weight', '.scales'): q_s, key.replace('.weight', '.biases'): q_b}" ], "awaits": false, "yields": false }, { "name": "update_model_index", "qualname": "update_model_index", "full_name": "scripts.add_mtp_weights_qwen35.update_model_index", "kind": "function", "signature": "def update_model_index(snapshot_dir: Path, mtp_keys: list[str])", "parameters": [ { "name": "snapshot_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "mtp_keys", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Update model.safetensors.index.json to include MTP weight keys.", "summary": "Update model.safetensors.index.json to include MTP weight keys.", "implementation": "Function `update_model_index` calls `index_path.exists`, `print`, `open`, `json.load`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 275, "end_line": 294, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L275-L294", "decorators": [], "calls": [ "index_path.exists", "print", "open", "json.load", "index.get", "json.dump", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "update_config", "qualname": "update_config", "full_name": "scripts.add_mtp_weights_qwen35.update_config", "kind": "function", "signature": "def update_config(snapshot_dir: Path)", "parameters": [ { "name": "snapshot_dir", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Update config.json to signal MTP availability.\n\nFor Qwen3.5, mtp_num_hidden_layers already exists in text_config.\nWe add num_nextn_predict_layers at top level for vllm-mlx compatibility.", "summary": "Update config.json to signal MTP availability.", "implementation": "Function `update_config` calls `open`, `json.load`, `config.get`, `text_config.get`.", "documented": true, "public": true, "addressable": true, "line": 297, "end_line": 322, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L297-L322", "decorators": [], "calls": [ "open", "json.load", "config.get", "text_config.get", "json.dump", "print" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "scripts.add_mtp_weights_qwen35.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "documented": false, "public": true, "addressable": true, "line": 325, "end_line": 466, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/add_mtp_weights_qwen35.py#L325-L466", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "find_snapshot_dir", "config_path.exists", "sys.exit", "open", "json.load", "config.get", "text_config.get", "mtp_file.exists", "mtp_file.stat", "Path", "download_dir.mkdir", "tempfile.mkdtemp", "fetch_shard_index", "identify_mtp_shards", "len", "sorted", "sum", "mtp_key_map.values", "download_shards", "p.exists", "extract_and_quantize_mtp_weights", "update_config", "shard_paths.values", "shard_path.unlink" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "scripts.check_docs_coverage", "path": "scripts/check_docs_coverage.py", "page_path": "reference/source/scripts/check_docs_coverage.md", "docstring": "Fail when source symbols or public explanations disappear from the docs.", "summary": "Fail when source symbols or public explanations disappear from the docs.", "line_count": 187, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/check_docs_coverage.py#L1-L187", "members": [ "main" ], "symbols": [ { "name": "main", "qualname": "main", "full_name": "scripts.check_docs_coverage.main", "kind": "function", "signature": "def main() -> int", "parameters": [], "return_annotation": "int", "docstring": "Validate module coverage, symbol coverage, and public docstring coverage.", "summary": "Validate module coverage, symbol coverage, and public docstring coverage.", "implementation": "Function `main` calls `build_inventory`, `build_repository_inventory`, `build_cli_inventory`, `render_cli_reference`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 17, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/check_docs_coverage.py#L17-L183", "decorators": [], "calls": [ "build_inventory", "build_repository_inventory", "build_cli_inventory", "render_cli_reference", "len", "set", "issues.append", "(REPOSITORY_ROOT / 'mkdocs.yml').read_text", "sum", "sorted", "docs_dir.rglob", "page.read_text", "content.startswith", "hand_written_pages.append", "any", "line.startswith", "content.splitlines", "content.lower", "page.relative_to", "http_reference.read_text", "next", "ast.parse", "(REPOSITORY_ROOT / server_module.path).read_text", "ast.walk", "isinstance", "ast.literal_eval", "decorator.func.attr.upper", "path.exists", "path.relative_to", "(docs_dir / 'reference' / 'python-symbols.md').read_text", "symbol_index.count", "print", "max" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "1", "0" ], "awaits": false, "yields": false } ] }, { "name": "scripts.docs_inventory", "path": "scripts/docs_inventory.py", "page_path": "reference/source/scripts/docs_inventory.md", "docstring": "Static source inventory shared by the documentation build tools.\n\nThe inventory uses Python's AST instead of importing :mod:`vllm_mlx`. This is\nimportant because the documentation build runs on Linux while the runtime\npackage depends on Apple Silicon and MLX.", "summary": "Static source inventory shared by the documentation build tools.", "line_count": 1182, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L1-L1182", "members": [ "REPOSITORY_URL", "SOURCE_BRANCH", "REPOSITORY_ROOT", "Parameter", "Symbol", "Module", "CLIOption", "_tracked_python_files", "module_name_for_path", "page_path_for_module", "_first_sentence", "_parameter_descriptions", "_function_parameters", "_class_parameters", "_owned_nodes", "_ordered_unique", "_short_expression", "_attribute_name", "_implementation_facts", "_function_signature", "_class_signature", "_defined_member_names", "scan_python_file", "build_inventory", "build_repository_inventory", "_literal_or_source", "scan_cli_options", "build_cli_inventory", "_escape_table_cell", "render_callable_signature", "_symbol_details_url", "render_contract_details", "render_source_map", "render_symbol_index", "render_module_page", "render_module_for_llms", "render_cli_reference" ], "symbols": [ { "name": "Parameter", "qualname": "Parameter", "full_name": "scripts.docs_inventory.Parameter", "kind": "class", "signature": "class Parameter", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "kind", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "annotation", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "default", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "required", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "Parameter", "docstring": "One explicit callable input reconstructed from the Python AST.", "summary": "One explicit callable input reconstructed from the Python AST.", "implementation": "Class `Parameter` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 24, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L24-L37", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "Parameter.to_dict", "full_name": "scripts.docs_inventory.Parameter.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict[str, object]", "parameters": [], "return_annotation": "dict[str, object]", "docstring": "Return a JSON-serializable representation of the parameter.", "summary": "Return a JSON-serializable representation of the parameter.", "implementation": "Method `Parameter.to_dict` calls `asdict`; returns `asdict(self)`.", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L34-L37", "decorators": [], "calls": [ "asdict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "asdict(self)" ], "awaits": false, "yields": false }, { "name": "Symbol", "qualname": "Symbol", "full_name": "scripts.docs_inventory.Symbol", "kind": "class", "signature": "class Symbol", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "qualname", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "full_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "kind", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "signature", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "parameters", "kind": "field", "annotation": "tuple[Parameter, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "return_annotation", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "docstring", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "summary", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "implementation", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "documented", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "public", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "addressable", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "line", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "end_line", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "source_url", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "decorators", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "calls", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "state_reads", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "state_writes", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "raises", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "return_expressions", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "awaits", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "yields", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "Symbol", "docstring": "A class, function, method, or nested definition found in source code.", "summary": "A class, function, method, or nested definition found in source code.", "implementation": "Class `Symbol` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 41, "end_line": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L41-L72", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "Symbol.to_dict", "full_name": "scripts.docs_inventory.Symbol.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict[str, object]", "parameters": [], "return_annotation": "dict[str, object]", "docstring": "Return a JSON-serializable representation of the symbol.", "summary": "Return a JSON-serializable representation of the symbol.", "implementation": "Method `Symbol.to_dict` calls `asdict`; returns `asdict(self)`.", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L69-L72", "decorators": [], "calls": [ "asdict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "asdict(self)" ], "awaits": false, "yields": false }, { "name": "Module", "qualname": "Module", "full_name": "scripts.docs_inventory.Module", "kind": "class", "signature": "class Module", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "path", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "page_path", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "docstring", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "summary", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "line_count", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "source_url", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "members", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "symbols", "kind": "field", "annotation": "tuple[Symbol, ...]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "Module", "docstring": "Documentation metadata for one tracked Python module.", "summary": "Documentation metadata for one tracked Python module.", "implementation": "Class `Module` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 76, "end_line": 94, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L76-L94", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "Module.to_dict", "full_name": "scripts.docs_inventory.Module.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict[str, object]", "parameters": [], "return_annotation": "dict[str, object]", "docstring": "Return a JSON-serializable representation of the module.", "summary": "Return a JSON-serializable representation of the module.", "implementation": "Method `Module.to_dict` calls `asdict`, `symbol.to_dict`; returns `payload`.", "documented": true, "public": true, "addressable": true, "line": 89, "end_line": 94, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L89-L94", "decorators": [], "calls": [ "asdict", "symbol.to_dict" ], "state_reads": [ "self.symbols" ], "state_writes": [], "raises": [], "return_expressions": [ "payload" ], "awaits": false, "yields": false }, { "name": "CLIOption", "qualname": "CLIOption", "full_name": "scripts.docs_inventory.CLIOption", "kind": "class", "signature": "class CLIOption", "parameters": [ { "name": "context", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "receiver", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "flags", "kind": "field", "annotation": "tuple[str, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "destination", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "default", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "required", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "choices", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "action", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "path", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "line", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "end_line", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "source_url", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "CLIOption", "docstring": "One argparse option declaration found in executable source.", "summary": "One argparse option declaration found in executable source.", "implementation": "Class `CLIOption` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 98, "end_line": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L98-L118", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "CLIOption.to_dict", "full_name": "scripts.docs_inventory.CLIOption.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict[str, object]", "parameters": [], "return_annotation": "dict[str, object]", "docstring": "Return a JSON-serializable representation of the option.", "summary": "Return a JSON-serializable representation of the option.", "implementation": "Method `CLIOption.to_dict` calls `asdict`; returns `asdict(self)`.", "documented": true, "public": true, "addressable": true, "line": 115, "end_line": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L115-L118", "decorators": [], "calls": [ "asdict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "asdict(self)" ], "awaits": false, "yields": false }, { "name": "_tracked_python_files", "qualname": "_tracked_python_files", "full_name": "scripts.docs_inventory._tracked_python_files", "kind": "function", "signature": "def _tracked_python_files(root: Path, source_roots: tuple[str, ...]=('vllm_mlx',)) -> list[Path]", "parameters": [ { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "source_roots", "kind": "positional or keyword", "annotation": "tuple[str, ...]", "default": "('vllm_mlx',)", "required": false, "description": "Optional positional or keyword input; defaults to `('vllm_mlx',)`." } ], "return_annotation": "list[Path]", "docstring": "Return tracked modules under selected roots, with a filesystem fallback.", "summary": "Return tracked modules under selected roots, with a filesystem fallback.", "implementation": "Function `_tracked_python_files` calls `subprocess.run`, `completed.stdout.splitlines`, `path.relative_to`, `sorted`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 121, "end_line": 154, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L121-L154", "decorators": [], "calls": [ "subprocess.run", "completed.stdout.splitlines", "path.relative_to", "sorted", "(root / source_root).rglob" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "sorted(tracked)", "sorted((path for source_root in source_roots for path in (root / source_root).rglob('*.py')))" ], "awaits": false, "yields": false }, { "name": "module_name_for_path", "qualname": "module_name_for_path", "full_name": "scripts.docs_inventory.module_name_for_path", "kind": "function", "signature": "def module_name_for_path(path: Path, root: Path=REPOSITORY_ROOT) -> str", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "REPOSITORY_ROOT", "required": false, "description": "Optional positional or keyword input; defaults to `REPOSITORY_ROOT`." } ], "return_annotation": "str", "docstring": "Convert a package source path into its importable dotted module name.", "summary": "Convert a package source path into its importable dotted module name.", "implementation": "Function `module_name_for_path` calls `path.relative_to(root).with_suffix`, `path.relative_to`, `list`, `parts.pop`; returns `'.'.join(parts)`.", "documented": true, "public": true, "addressable": true, "line": 157, "end_line": 164, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L157-L164", "decorators": [], "calls": [ "path.relative_to(root).with_suffix", "path.relative_to", "list", "parts.pop", "'.'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'.'.join(parts)" ], "awaits": false, "yields": false }, { "name": "page_path_for_module", "qualname": "page_path_for_module", "full_name": "scripts.docs_inventory.page_path_for_module", "kind": "function", "signature": "def page_path_for_module(path: Path, root: Path=REPOSITORY_ROOT) -> Path", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "REPOSITORY_ROOT", "required": false, "description": "Optional positional or keyword input; defaults to `REPOSITORY_ROOT`." } ], "return_annotation": "Path", "docstring": "Return the generated documentation path for a package source file.", "summary": "Return the generated documentation path for a package source file.", "implementation": "Function `page_path_for_module` calls `path.relative_to(root).with_suffix`, `path.relative_to`, `relative.with_name`, `Path`; returns `Path('reference') / section / relative`.", "documented": true, "public": true, "addressable": true, "line": 167, "end_line": 174, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L167-L174", "decorators": [], "calls": [ "path.relative_to(root).with_suffix", "path.relative_to", "relative.with_name", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Path('reference') / section / relative" ], "awaits": false, "yields": false }, { "name": "_first_sentence", "qualname": "_first_sentence", "full_name": "scripts.docs_inventory._first_sentence", "kind": "function", "signature": "def _first_sentence(docstring: str) -> str", "parameters": [ { "name": "docstring", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Return a compact first sentence or line from a docstring.", "summary": "Return a compact first sentence or line from a docstring.", "implementation": "Function `_first_sentence` calls `' '.join`, `docstring.strip().split`, `docstring.strip`, `re.search`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 177, "end_line": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L177-L184", "decorators": [], "calls": [ "' '.join", "docstring.strip().split", "docstring.strip", "re.search", "match.start" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''", "text[:match.start()] if match else text" ], "awaits": false, "yields": false }, { "name": "_parameter_descriptions", "qualname": "_parameter_descriptions", "full_name": "scripts.docs_inventory._parameter_descriptions", "kind": "function", "signature": "def _parameter_descriptions(docstring: str) -> dict[str, str]", "parameters": [ { "name": "docstring", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, str]", "docstring": "Extract Google, NumPy, and Sphinx parameter descriptions.", "summary": "Extract Google, NumPy, and Sphinx parameter descriptions.", "implementation": "Function `_parameter_descriptions` calls `docstring.splitlines`, `re.match`, `sphinx_match.group(1).lstrip`, `sphinx_match.group`; returns `descriptions`.", "documented": true, "public": false, "addressable": true, "line": 187, "end_line": 266, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L187-L266", "decorators": [], "calls": [ "docstring.splitlines", "re.match", "sphinx_match.group(1).lstrip", "sphinx_match.group", "sphinx_match.group(2).strip", "next", "enumerate", "line.strip", "line[:1].isspace", "stripped.endswith", "name.strip().lstrip", "name.strip", "item.group(1).split", "item.group", "item.group(2).strip", "' '.join", "set", "lines[index + 1].strip", "item_line.strip", "item_line[:1].isspace", "descriptions.setdefault" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "descriptions" ], "awaits": false, "yields": false }, { "name": "_function_parameters", "qualname": "_function_parameters", "full_name": "scripts.docs_inventory._function_parameters", "kind": "function", "signature": "def _function_parameters(node: ast.FunctionDef | ast.AsyncFunctionDef, docstring: str) -> tuple[Parameter, ...]", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.FunctionDef | ast.AsyncFunctionDef", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "docstring", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[Parameter, ...]", "docstring": "Return the callable inputs, annotations, defaults, and descriptions.", "summary": "Return the callable inputs, annotations, defaults, and descriptions.", "implementation": "Function `_function_parameters` calls `_parameter_descriptions`, `len`, `list`, `enumerate`; returns `tuple(parameters)`.", "documented": true, "public": false, "addressable": true, "line": 269, "end_line": 353, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L269-L353", "decorators": [], "calls": [ "_parameter_descriptions", "len", "list", "enumerate", "zip", "append_parameter", "tuple" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tuple(parameters)" ], "awaits": false, "yields": false }, { "name": "append_parameter", "qualname": "_function_parameters.append_parameter", "full_name": "scripts.docs_inventory._function_parameters.append_parameter", "kind": "nested function", "signature": "def append_parameter(argument: ast.arg, *, kind: str, default_node: ast.AST | None, required: bool, prefix: str='') -> None", "parameters": [ { "name": "argument", "kind": "positional or keyword", "annotation": "ast.arg", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "kind", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "default_node", "kind": "keyword-only", "annotation": "ast.AST | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "required", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "prefix", "kind": "keyword-only", "annotation": "str", "default": "''", "required": false, "description": "Optional keyword-only input; defaults to `''`." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `_function_parameters.append_parameter` calls `_short_expression`, `descriptions.get`, `parameters.append`, `Parameter`; returns `None`.", "implementation": "Nested Function `_function_parameters.append_parameter` calls `_short_expression`, `descriptions.get`, `parameters.append`, `Parameter`; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 282, "end_line": 314, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L282-L314", "decorators": [], "calls": [ "_short_expression", "descriptions.get", "parameters.append", "Parameter" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_class_parameters", "qualname": "_class_parameters", "full_name": "scripts.docs_inventory._class_parameters", "kind": "function", "signature": "def _class_parameters(node: ast.ClassDef, docstring: str) -> tuple[Parameter, ...]", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.ClassDef", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "docstring", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[Parameter, ...]", "docstring": "Return constructor inputs from ``__init__`` or declarative fields.", "summary": "Return constructor inputs from ``__init__`` or declarative fields.", "implementation": "Function `_class_parameters` calls `next`, `isinstance`, `ast.get_docstring`, `_function_parameters`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 356, "end_line": 408, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L356-L408", "decorators": [], "calls": [ "next", "isinstance", "ast.get_docstring", "_function_parameters", "'\\n'.join", "ast.unparse(item).split", "ast.unparse", "ast.unparse(base).rsplit", "_parameter_descriptions", "_short_expression", "parameters.append", "Parameter", "descriptions.get", "tuple" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_function_parameters(initializer, '\\n'.join((part for part in (docstring, initializer_docstring) if part)))", "()", "tuple(parameters)" ], "awaits": false, "yields": false }, { "name": "_owned_nodes", "qualname": "_owned_nodes", "full_name": "scripts.docs_inventory._owned_nodes", "kind": "function", "signature": "def _owned_nodes(node: ast.AST) -> Iterable[ast.AST]", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.AST", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Iterable[ast.AST]", "docstring": "Walk implementation nodes without attributing nested bodies to parents.", "summary": "Walk implementation nodes without attributing nested bodies to parents.", "implementation": "Function `_owned_nodes` calls `getattr`, `list`, `reversed`, `stack.pop`; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 411, "end_line": 423, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L411-L423", "decorators": [], "calls": [ "getattr", "list", "reversed", "stack.pop", "isinstance", "stack.extend", "ast.iter_child_nodes" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "_ordered_unique", "qualname": "_ordered_unique", "full_name": "scripts.docs_inventory._ordered_unique", "kind": "function", "signature": "def _ordered_unique(values: Iterable[str]) -> tuple[str, ...]", "parameters": [ { "name": "values", "kind": "positional or keyword", "annotation": "Iterable[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, ...]", "docstring": "Return non-empty strings once while preserving their source order.", "summary": "Return non-empty strings once while preserving their source order.", "implementation": "Function `_ordered_unique` calls `tuple`, `dict.fromkeys`; returns `tuple(dict.fromkeys((value for value in values if value)))`.", "documented": true, "public": false, "addressable": true, "line": 426, "end_line": 429, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L426-L429", "decorators": [], "calls": [ "tuple", "dict.fromkeys" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tuple(dict.fromkeys((value for value in values if value)))" ], "awaits": false, "yields": false }, { "name": "_short_expression", "qualname": "_short_expression", "full_name": "scripts.docs_inventory._short_expression", "kind": "function", "signature": "def _short_expression(node: ast.AST | None, *, limit: int=120) -> str", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.AST | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "limit", "kind": "keyword-only", "annotation": "int", "default": "120", "required": false, "description": "Optional keyword-only input; defaults to `120`." } ], "return_annotation": "str", "docstring": "Render an AST expression without allowing one fact to dominate output.", "summary": "Render an AST expression without allowing one fact to dominate output.", "implementation": "Function `_short_expression` calls `' '.join`, `ast.unparse(node).split`, `ast.unparse`, `len`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 432, "end_line": 438, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L432-L438", "decorators": [], "calls": [ "' '.join", "ast.unparse(node).split", "ast.unparse", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'None'", "value if len(value) <= limit else value[:limit - 1] + '…'" ], "awaits": false, "yields": false }, { "name": "_attribute_name", "qualname": "_attribute_name", "full_name": "scripts.docs_inventory._attribute_name", "kind": "function", "signature": "def _attribute_name(node: ast.Attribute) -> str", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.Attribute", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Return tracked ``self`` or ``cls`` attribute access, if applicable.", "summary": "Return tracked ``self`` or ``cls`` attribute access, if applicable.", "implementation": "Function `_attribute_name` calls `isinstance`, `parts.append`, `'.'.join`, `reversed`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 441, "end_line": 452, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L441-L452", "decorators": [], "calls": [ "isinstance", "parts.append", "'.'.join", "reversed" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'.'.join(reversed(parts))", "''" ], "awaits": false, "yields": false }, { "name": "_implementation_facts", "qualname": "_implementation_facts", "full_name": "scripts.docs_inventory._implementation_facts", "kind": "function", "signature": "def _implementation_facts(node: ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef, *, kind: str, qualname: str) -> dict[str, object]", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "kind", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "qualname", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "dict[str, object]", "docstring": "Extract conservative behavioral facts from one definition's own body.", "summary": "Extract conservative behavioral facts from one definition's own body.", "implementation": "Function `_implementation_facts` calls `_ordered_unique`, `ast.unparse`, `isinstance`, `clauses.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 455, "end_line": 548, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L455-L548", "decorators": [], "calls": [ "_ordered_unique", "ast.unparse", "isinstance", "clauses.append", "', '.join", "len", "kind.title", "' and '.join", "list", "_owned_nodes", "_short_expression", "_attribute_name", "any", "'; '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'implementation': implementation, 'decorators': decorators, 'calls': (), 'state_reads': (), 'state_writes': (), 'raise…", "{'implementation': implementation, 'decorators': decorators, 'calls': calls, 'state_reads': state_reads, 'state_writes'…" ], "awaits": false, "yields": false }, { "name": "_function_signature", "qualname": "_function_signature", "full_name": "scripts.docs_inventory._function_signature", "kind": "function", "signature": "def _function_signature(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.FunctionDef | ast.AsyncFunctionDef", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render a stable function signature without importing its module.", "summary": "Render a stable function signature without importing its module.", "implementation": "Function `_function_signature` calls `isinstance`, `ast.unparse`; returns `f'{prefix} {node.name}({ast.unparse(node.args)}){returns}'`.", "documented": true, "public": false, "addressable": true, "line": 551, "end_line": 556, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L551-L556", "decorators": [], "calls": [ "isinstance", "ast.unparse" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'{prefix} {node.name}({ast.unparse(node.args)}){returns}'" ], "awaits": false, "yields": false }, { "name": "_class_signature", "qualname": "_class_signature", "full_name": "scripts.docs_inventory._class_signature", "kind": "function", "signature": "def _class_signature(node: ast.ClassDef) -> str", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.ClassDef", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render a class declaration and its bases from the AST.", "summary": "Render a class declaration and its bases from the AST.", "implementation": "Function `_class_signature` calls `ast.unparse`, `arguments.extend`, `', '.join`; returns `f'class {node.name}{suffix}'`.", "documented": true, "public": false, "addressable": true, "line": 559, "end_line": 565, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L559-L565", "decorators": [], "calls": [ "ast.unparse", "arguments.extend", "', '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'class {node.name}{suffix}'" ], "awaits": false, "yields": false }, { "name": "_defined_member_names", "qualname": "_defined_member_names", "full_name": "scripts.docs_inventory._defined_member_names", "kind": "function", "signature": "def _defined_member_names(tree: ast.Module) -> tuple[str, ...]", "parameters": [ { "name": "tree", "kind": "positional or keyword", "annotation": "ast.Module", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, ...]", "docstring": "Return names defined directly by a module in source order.", "summary": "Return names defined directly by a module in source order.", "implementation": "Function `_defined_member_names` calls `isinstance`, `names.append`, `tuple`, `dict.fromkeys`; returns `tuple(dict.fromkeys(names))`.", "documented": true, "public": false, "addressable": true, "line": 568, "end_line": 584, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L568-L584", "decorators": [], "calls": [ "isinstance", "names.append", "tuple", "dict.fromkeys" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tuple(dict.fromkeys(names))" ], "awaits": false, "yields": false }, { "name": "scan_python_file", "qualname": "scan_python_file", "full_name": "scripts.docs_inventory.scan_python_file", "kind": "function", "signature": "def scan_python_file(path: Path, root: Path=REPOSITORY_ROOT) -> Module", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "REPOSITORY_ROOT", "required": false, "description": "Optional positional or keyword input; defaults to `REPOSITORY_ROOT`." } ], "return_annotation": "Module", "docstring": "Parse one Python file into module and symbol documentation metadata.", "summary": "Parse one Python file into module and symbol documentation metadata.", "implementation": "Function `scan_python_file` calls `path.read_text`, `ast.parse`, `str`, `ast.walk`; returns `Module(name=module_name, path=relative_path, page_path=page_path_for_module(path, root).as_posix(), docstring=module_do…`.", "documented": true, "public": true, "addressable": true, "line": 587, "end_line": 712, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L587-L712", "decorators": [], "calls": [ "path.read_text", "ast.parse", "str", "ast.walk", "ast.iter_child_nodes", "module_name_for_path", "isinstance", "parents.get", "ancestors.append", "ancestors.reverse", "any", "'.'.join", "all", "part.startswith", "ast.get_docstring", "_class_signature", "_class_parameters", "_function_signature", "_function_parameters", "_short_expression", "getattr", "path.relative_to(root).as_posix", "path.relative_to", "_implementation_facts", "symbols.append", "Symbol", "_first_sentence", "bool", "symbols.sort", "len", "source.splitlines", "Module", "page_path_for_module(path, root).as_posix", "page_path_for_module", "max", "_defined_member_names", "tuple" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Module(name=module_name, path=relative_path, page_path=page_path_for_module(path, root).as_posix(), docstring=module_do…" ], "awaits": false, "yields": false }, { "name": "build_inventory", "qualname": "build_inventory", "full_name": "scripts.docs_inventory.build_inventory", "kind": "function", "signature": "def build_inventory(root: Path=REPOSITORY_ROOT, source_roots: tuple[str, ...]=('vllm_mlx',)) -> list[Module]", "parameters": [ { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "REPOSITORY_ROOT", "required": false, "description": "Optional positional or keyword input; defaults to `REPOSITORY_ROOT`." }, { "name": "source_roots", "kind": "positional or keyword", "annotation": "tuple[str, ...]", "default": "('vllm_mlx',)", "required": false, "description": "Optional positional or keyword input; defaults to `('vllm_mlx',)`." } ], "return_annotation": "list[Module]", "docstring": "Build static source inventory for the selected repository roots.", "summary": "Build static source inventory for the selected repository roots.", "implementation": "Function `build_inventory` calls `scan_python_file`, `_tracked_python_files`; returns `[scan_python_file(path, root) for path in _tracked_python_files(root, source_roots)]`.", "documented": true, "public": true, "addressable": true, "line": 715, "end_line": 724, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L715-L724", "decorators": [], "calls": [ "scan_python_file", "_tracked_python_files" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[scan_python_file(path, root) for path in _tracked_python_files(root, source_roots)]" ], "awaits": false, "yields": false }, { "name": "build_repository_inventory", "qualname": "build_repository_inventory", "full_name": "scripts.docs_inventory.build_repository_inventory", "kind": "function", "signature": "def build_repository_inventory(root: Path=REPOSITORY_ROOT) -> list[Module]", "parameters": [ { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "REPOSITORY_ROOT", "required": false, "description": "Optional positional or keyword input; defaults to `REPOSITORY_ROOT`." } ], "return_annotation": "list[Module]", "docstring": "Inventory runtime, documentation tools, and executable examples.", "summary": "Inventory runtime, documentation tools, and executable examples.", "implementation": "Function `build_repository_inventory` calls `build_inventory`; returns `build_inventory(root, ('vllm_mlx', 'scripts', 'examples'))`.", "documented": true, "public": true, "addressable": true, "line": 727, "end_line": 730, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L727-L730", "decorators": [], "calls": [ "build_inventory" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "build_inventory(root, ('vllm_mlx', 'scripts', 'examples'))" ], "awaits": false, "yields": false }, { "name": "_literal_or_source", "qualname": "_literal_or_source", "full_name": "scripts.docs_inventory._literal_or_source", "kind": "function", "signature": "def _literal_or_source(node: ast.AST | None, default: str='') -> str", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "ast.AST | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "default", "kind": "positional or keyword", "annotation": "str", "default": "''", "required": false, "description": "Optional positional or keyword input; defaults to `''`." } ], "return_annotation": "str", "docstring": "Render a simple literal cleanly and preserve expressions as source.", "summary": "Render a simple literal cleanly and preserve expressions as source.", "implementation": "Function `_literal_or_source` calls `ast.literal_eval`, `' '.join`, `ast.unparse(node).split`, `ast.unparse`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 733, "end_line": 744, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L733-L744", "decorators": [], "calls": [ "ast.literal_eval", "' '.join", "ast.unparse(node).split", "ast.unparse", "isinstance", "value.split", "repr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "default", "' '.join(ast.unparse(node).split())", "' '.join(value.split())", "repr(value)" ], "awaits": false, "yields": false }, { "name": "scan_cli_options", "qualname": "scan_cli_options", "full_name": "scripts.docs_inventory.scan_cli_options", "kind": "function", "signature": "def scan_cli_options(paths: Iterable[Path], root: Path=REPOSITORY_ROOT) -> list[CLIOption]", "parameters": [ { "name": "paths", "kind": "positional or keyword", "annotation": "Iterable[Path]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "REPOSITORY_ROOT", "required": false, "description": "Optional positional or keyword input; defaults to `REPOSITORY_ROOT`." } ], "return_annotation": "list[CLIOption]", "docstring": "Extract every argparse ``add_argument`` call from selected source files.", "summary": "Extract every argparse ``add_argument`` call from selected source files.", "implementation": "Function `scan_cli_options` calls `sorted`, `path.read_text`, `ast.parse`, `str`; returns `sorted(options, key=lambda option: (option.path, option.line))`.", "documented": true, "public": true, "addressable": true, "line": 747, "end_line": 817, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L747-L817", "decorators": [], "calls": [ "sorted", "path.read_text", "ast.parse", "str", "ast.walk", "ast.iter_child_nodes", "module_name_for_path", "path.relative_to(root).as_posix", "path.relative_to", "isinstance", "tuple", "_literal_or_source", "keywords.get", "next", "flag.startswith", "preferred.lstrip('-').replace", "preferred.lstrip", "parents.get", "ancestors.append", "ancestors.reverse", "'.'.join", "' '.join", "ast.unparse(node.func.value).split", "ast.unparse", "getattr", "options.append", "CLIOption", "flags[0].startswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "sorted(options, key=lambda option: (option.path, option.line))" ], "awaits": false, "yields": false }, { "name": "build_cli_inventory", "qualname": "build_cli_inventory", "full_name": "scripts.docs_inventory.build_cli_inventory", "kind": "function", "signature": "def build_cli_inventory(root: Path=REPOSITORY_ROOT) -> list[CLIOption]", "parameters": [ { "name": "root", "kind": "positional or keyword", "annotation": "Path", "default": "REPOSITORY_ROOT", "required": false, "description": "Optional positional or keyword input; defaults to `REPOSITORY_ROOT`." } ], "return_annotation": "list[CLIOption]", "docstring": "Build the complete argparse option inventory for executable source.", "summary": "Build the complete argparse option inventory for executable source.", "implementation": "Function `build_cli_inventory` calls `_tracked_python_files`, `scan_cli_options`; returns `scan_cli_options(paths, root)`.", "documented": true, "public": true, "addressable": true, "line": 820, "end_line": 824, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L820-L824", "decorators": [], "calls": [ "_tracked_python_files", "scan_cli_options" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "scan_cli_options(paths, root)" ], "awaits": false, "yields": false }, { "name": "_escape_table_cell", "qualname": "_escape_table_cell", "full_name": "scripts.docs_inventory._escape_table_cell", "kind": "function", "signature": "def _escape_table_cell(value: str) -> str", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Escape text for a compact Markdown table cell.", "summary": "Escape text for a compact Markdown table cell.", "implementation": "Function `_escape_table_cell` calls `' '.join`, `value.replace('|', '\\\\|').split`, `value.replace`; returns `' '.join(value.replace('|', '\\\\|').split())`.", "documented": true, "public": false, "addressable": true, "line": 827, "end_line": 830, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L827-L830", "decorators": [], "calls": [ "' '.join", "value.replace('|', '\\\\|').split", "value.replace" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "' '.join(value.replace('|', '\\\\|').split())" ], "awaits": false, "yields": false }, { "name": "render_callable_signature", "qualname": "render_callable_signature", "full_name": "scripts.docs_inventory.render_callable_signature", "kind": "function", "signature": "def render_callable_signature(symbol: Symbol, *, qualified: bool=False) -> str", "parameters": [ { "name": "symbol", "kind": "positional or keyword", "annotation": "Symbol", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "qualified", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." } ], "return_annotation": "str", "docstring": "Render a reader-facing signature without implicit ``self`` or ``cls``.", "summary": "Render a reader-facing signature without implicit ``self`` or ``cls``.", "implementation": "Function `render_callable_signature` calls `sum`, `any`, `pieces.append`, `symbol.signature.startswith`; returns `signature`.", "documented": true, "public": true, "addressable": true, "line": 833, "end_line": 866, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L833-L866", "decorators": [], "calls": [ "sum", "any", "pieces.append", "symbol.signature.startswith", "', '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "signature" ], "awaits": false, "yields": false }, { "name": "_symbol_details_url", "qualname": "_symbol_details_url", "full_name": "scripts.docs_inventory._symbol_details_url", "kind": "function", "signature": "def _symbol_details_url(module: Module, symbol: Symbol) -> str", "parameters": [ { "name": "module", "kind": "positional or keyword", "annotation": "Module", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "symbol", "kind": "positional or keyword", "annotation": "Symbol", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Return the generated details URL or source fallback for one symbol.", "summary": "Return the generated details URL or source fallback for one symbol.", "implementation": "Function `_symbol_details_url` calls `Path(module.page_path).relative_to`, `Path`, `relative.parent.as_posix`, `relative.with_suffix('').as_posix`; returns `f'../{page_url}#contract-{symbol.full_name}'`.", "documented": true, "public": false, "addressable": true, "line": 869, "end_line": 877, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L869-L877", "decorators": [], "calls": [ "Path(module.page_path).relative_to", "Path", "relative.parent.as_posix", "relative.with_suffix('').as_posix", "relative.with_suffix" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'../{page_url}#contract-{symbol.full_name}'" ], "awaits": false, "yields": false }, { "name": "render_contract_details", "qualname": "render_contract_details", "full_name": "scripts.docs_inventory.render_contract_details", "kind": "function", "signature": "def render_contract_details(module: Module) -> str", "parameters": [ { "name": "module", "kind": "positional or keyword", "annotation": "Module", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render explicit inputs and behavior for every definition in a module.", "summary": "Render explicit inputs and behavior for every definition in a module.", "implementation": "Function `render_contract_details` calls `lines.extend`, `render_callable_signature`, `lines.append`, `_escape_table_cell`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 880, "end_line": 959, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L880-L959", "decorators": [], "calls": [ "lines.extend", "render_callable_signature", "lines.append", "_escape_table_cell", "'; '.join", "', '.join", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'This module does not declare classes or functions.\\n'", "'\\n'.join(lines)" ], "awaits": false, "yields": false }, { "name": "render_source_map", "qualname": "render_source_map", "full_name": "scripts.docs_inventory.render_source_map", "kind": "function", "signature": "def render_source_map(module: Module) -> str", "parameters": [ { "name": "module", "kind": "positional or keyword", "annotation": "Module", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render a line-precise source table for every definition in a module.", "summary": "Render a line-precise source table for every definition in a module.", "implementation": "Function `render_source_map` calls `lines.append`, `_escape_table_cell`, `render_callable_signature`, `'\\n'.join`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 962, "end_line": 981, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L962-L981", "decorators": [], "calls": [ "lines.append", "_escape_table_cell", "render_callable_signature", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'This module does not declare classes or functions.\\n'", "'\\n'.join(lines) + '\\n'" ], "awaits": false, "yields": false }, { "name": "render_symbol_index", "qualname": "render_symbol_index", "full_name": "scripts.docs_inventory.render_symbol_index", "kind": "function", "signature": "def render_symbol_index(modules: list[Module]) -> str", "parameters": [ { "name": "modules", "kind": "positional or keyword", "annotation": "list[Module]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render a filterable index of every runtime class and callable.", "summary": "Render a filterable index of every runtime class and callable.", "implementation": "Function `render_symbol_index` calls `sorted`, `len`, `render_callable_signature`, `' '.join((symbol.full_name, symbol.kind, signature, symbol.summary)).casefold`; returns `'\\n'.join(lines)`.", "documented": true, "public": true, "addressable": true, "line": 984, "end_line": 1039, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L984-L1039", "decorators": [], "calls": [ "sorted", "len", "render_callable_signature", "' '.join((symbol.full_name, symbol.kind, signature, symbol.summary)).casefold", "' '.join", "_symbol_details_url", "lines.extend", "html.escape", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'\\n'.join(lines)" ], "awaits": false, "yields": false }, { "name": "render_module_page", "qualname": "render_module_page", "full_name": "scripts.docs_inventory.render_module_page", "kind": "function", "signature": "def render_module_page(module: Module) -> str", "parameters": [ { "name": "module", "kind": "positional or keyword", "annotation": "Module", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render the generated MkDocs page for one Python module.", "summary": "Render the generated MkDocs page for one Python module.", "implementation": "Function `render_module_page` calls `lines.extend`, `lines.append`, `render_contract_details(module).rstrip`, `render_contract_details`; returns `'\\n'.join(lines)`.", "documented": true, "public": true, "addressable": true, "line": 1042, "end_line": 1086, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L1042-L1086", "decorators": [], "calls": [ "lines.extend", "lines.append", "render_contract_details(module).rstrip", "render_contract_details", "render_source_map(module).rstrip", "render_source_map", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'\\n'.join(lines)" ], "awaits": false, "yields": false }, { "name": "render_module_for_llms", "qualname": "render_module_for_llms", "full_name": "scripts.docs_inventory.render_module_for_llms", "kind": "function", "signature": "def render_module_for_llms(module: Module) -> str", "parameters": [ { "name": "module", "kind": "positional or keyword", "annotation": "Module", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render a self-contained plain-Markdown API record for language models.", "summary": "Render a self-contained plain-Markdown API record for language models.", "implementation": "Function `render_module_for_llms` calls `lines.extend`, `lines.append`, `', '.join`, `'; '.join`; returns `'\\n'.join(lines) + '\\n'`.", "documented": true, "public": true, "addressable": true, "line": 1089, "end_line": 1144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L1089-L1144", "decorators": [], "calls": [ "lines.extend", "lines.append", "', '.join", "'; '.join", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'\\n'.join(lines) + '\\n'" ], "awaits": false, "yields": false }, { "name": "render_cli_reference", "qualname": "render_cli_reference", "full_name": "scripts.docs_inventory.render_cli_reference", "kind": "function", "signature": "def render_cli_reference(options: list[CLIOption]) -> str", "parameters": [ { "name": "options", "kind": "positional or keyword", "annotation": "list[CLIOption]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render every discovered argparse option as a line-precise reference.", "summary": "Render every discovered argparse option as a line-precise reference.", "implementation": "Function `render_cli_reference` calls `lines.extend`, `', '.join`, `str(option.required).lower`, `str`; returns `'\\n'.join(lines)`.", "documented": true, "public": true, "addressable": true, "line": 1147, "end_line": 1182, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/docs_inventory.py#L1147-L1182", "decorators": [], "calls": [ "lines.extend", "', '.join", "str(option.required).lower", "str", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'\\n'.join(lines)" ], "awaits": false, "yields": false } ] }, { "name": "scripts.gen_api_reference", "path": "scripts/gen_api_reference.py", "page_path": "reference/source/scripts/gen_api_reference.md", "docstring": "Synchronize the exhaustive static API reference with the Python source tree.", "summary": "Synchronize the exhaustive static API reference with the Python source tree.", "line_count": 103, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/gen_api_reference.py#L1-L103", "members": [ "GENERATED_HEADER", "expected_pages", "generated_pages_on_disk", "check", "write", "main" ], "symbols": [ { "name": "expected_pages", "qualname": "expected_pages", "full_name": "scripts.gen_api_reference.expected_pages", "kind": "function", "signature": "def expected_pages() -> dict[Path, str]", "parameters": [], "return_annotation": "dict[Path, str]", "docstring": "Return every generated reference path and its expected contents.", "summary": "Return every generated reference path and its expected contents.", "implementation": "Function `expected_pages` calls `build_inventory`, `render_module_page`, `build_repository_inventory`, `render_cli_reference`; returns `pages`.", "documented": true, "public": true, "addressable": true, "line": 23, "end_line": 38, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/gen_api_reference.py#L23-L38", "decorators": [], "calls": [ "build_inventory", "render_module_page", "build_repository_inventory", "render_cli_reference", "build_cli_inventory", "render_symbol_index" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "pages" ], "awaits": false, "yields": false }, { "name": "generated_pages_on_disk", "qualname": "generated_pages_on_disk", "full_name": "scripts.gen_api_reference.generated_pages_on_disk", "kind": "function", "signature": "def generated_pages_on_disk() -> set[Path]", "parameters": [], "return_annotation": "set[Path]", "docstring": "Return generated reference pages currently present in the docs tree.", "summary": "Return generated reference pages currently present in the docs tree.", "implementation": "Function `generated_pages_on_disk` calls `docs_dir.rglob`, `path.read_text(encoding='utf-8').startswith`, `path.read_text`; returns `{path for path in docs_dir.rglob('*.md') if path.read_text(encoding='utf-8').startswith(GENERATED_HEADER)}`.", "documented": true, "public": true, "addressable": true, "line": 41, "end_line": 49, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/gen_api_reference.py#L41-L49", "decorators": [], "calls": [ "docs_dir.rglob", "path.read_text(encoding='utf-8').startswith", "path.read_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{path for path in docs_dir.rglob('*.md') if path.read_text(encoding='utf-8').startswith(GENERATED_HEADER)}" ], "awaits": false, "yields": false }, { "name": "check", "qualname": "check", "full_name": "scripts.gen_api_reference.check", "kind": "function", "signature": "def check() -> int", "parameters": [], "return_annotation": "int", "docstring": "Report missing, stale, or unexpected generated pages.", "summary": "Report missing, stale, or unexpected generated pages.", "implementation": "Function `check` calls `expected_pages`, `generated_pages_on_disk`, `expected.items`, `path.exists`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 52, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/gen_api_reference.py#L52-L73", "decorators": [], "calls": [ "expected_pages", "generated_pages_on_disk", "expected.items", "path.exists", "issues.append", "path.relative_to", "path.read_text", "sorted", "expected.keys", "print", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "1", "0" ], "awaits": false, "yields": false }, { "name": "write", "qualname": "write", "full_name": "scripts.gen_api_reference.write", "kind": "function", "signature": "def write() -> int", "parameters": [], "return_annotation": "int", "docstring": "Write every reference page and remove obsolete generated pages.", "summary": "Write every reference page and remove obsolete generated pages.", "implementation": "Function `write` calls `expected_pages`, `generated_pages_on_disk`, `expected.keys`, `path.unlink`; returns `0`.", "documented": true, "public": true, "addressable": true, "line": 76, "end_line": 86, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/gen_api_reference.py#L76-L86", "decorators": [], "calls": [ "expected_pages", "generated_pages_on_disk", "expected.keys", "path.unlink", "expected.items", "path.parent.mkdir", "path.write_text", "print", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "scripts.gen_api_reference.main", "kind": "function", "signature": "def main() -> int", "parameters": [], "return_annotation": "int", "docstring": "Parse the synchronization mode and update or verify generated pages.", "summary": "Parse the synchronization mode and update or verify generated pages.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `check`; returns `check() if args.check else write()`.", "documented": true, "public": true, "addressable": true, "line": 89, "end_line": 99, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/gen_api_reference.py#L89-L99", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "check", "write" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "check() if args.check else write()" ], "awaits": false, "yields": false } ] }, { "name": "scripts.mkdocs_hooks", "path": "scripts/mkdocs_hooks.py", "page_path": "reference/source/scripts/mkdocs_hooks.md", "docstring": "MkDocs hooks that publish machine-readable documentation artifacts.", "summary": "MkDocs hooks that publish machine-readable documentation artifacts.", "line_count": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/mkdocs_hooks.py#L1-L225", "members": [ "SOURCE_BRANCH_URL", "_source_revision", "_pin_source_links", "on_page_markdown", "on_post_page", "_markdown_documents", "on_post_build" ], "symbols": [ { "name": "_source_revision", "qualname": "_source_revision", "full_name": "scripts.mkdocs_hooks._source_revision", "kind": "function", "signature": "def _source_revision() -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the immutable commit represented by this documentation build.", "summary": "Return the immutable commit represented by this documentation build.", "implementation": "Function `_source_revision` calls `os.environ.get`, `re.fullmatch`, `candidate.lower`, `subprocess.run`; can raise `ValueError`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 27, "end_line": 46, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/mkdocs_hooks.py#L27-L46", "decorators": [ "lru_cache(maxsize=1)" ], "calls": [ "os.environ.get", "re.fullmatch", "candidate.lower", "subprocess.run", "completed.stdout.strip", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "candidate.lower()", "revision" ], "awaits": false, "yields": false }, { "name": "_pin_source_links", "qualname": "_pin_source_links", "full_name": "scripts.mkdocs_hooks._pin_source_links", "kind": "function", "signature": "def _pin_source_links(text: str, revision: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "revision", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Replace mutable gh-pages source links with one commit permalink.", "summary": "Replace mutable gh-pages source links with one commit permalink.", "implementation": "Function `_pin_source_links` calls `text.replace`; returns `text.replace(SOURCE_BRANCH_URL, f'https://github.com/waybarrios/vllm-mlx/blob/{revision}/')`.", "documented": true, "public": false, "addressable": true, "line": 49, "end_line": 55, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/mkdocs_hooks.py#L49-L55", "decorators": [], "calls": [ "text.replace" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text.replace(SOURCE_BRANCH_URL, f'https://github.com/waybarrios/vllm-mlx/blob/{revision}/')" ], "awaits": false, "yields": false }, { "name": "on_page_markdown", "qualname": "on_page_markdown", "full_name": "scripts.mkdocs_hooks.on_page_markdown", "kind": "function", "signature": "def on_page_markdown(markdown: str, **kwargs) -> str", "parameters": [ { "name": "markdown", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "str", "docstring": "Pin every rendered GitHub source link to the build commit.", "summary": "Pin every rendered GitHub source link to the build commit.", "implementation": "Function `on_page_markdown` calls `_pin_source_links`, `_source_revision`; returns `_pin_source_links(markdown, _source_revision())`.", "documented": true, "public": true, "addressable": true, "line": 58, "end_line": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/mkdocs_hooks.py#L58-L62", "decorators": [], "calls": [ "_pin_source_links", "_source_revision" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_pin_source_links(markdown, _source_revision())" ], "awaits": false, "yields": false }, { "name": "on_post_page", "qualname": "on_post_page", "full_name": "scripts.mkdocs_hooks.on_post_page", "kind": "function", "signature": "def on_post_page(output: str, page=None, **kwargs) -> str", "parameters": [ { "name": "output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "page", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "str", "docstring": "Normalize search alternates and localized homepage presentation.", "summary": "Normalize search alternates and localized homepage presentation.", "implementation": "Function `on_post_page` calls `getattr`, `re.sub`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 65, "end_line": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/mkdocs_hooks.py#L65-L89", "decorators": [], "calls": [ "getattr", "re.sub" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "output", "re.sub('\\\\s*]*\\\\brel=\\\\\"edit\\\\\")[^>]*>.*?', '', output, count=1, flags=re.DOTALL)" ], "awaits": false, "yields": false }, { "name": "_markdown_documents", "qualname": "_markdown_documents", "full_name": "scripts.mkdocs_hooks._markdown_documents", "kind": "function", "signature": "def _markdown_documents() -> list[Path]", "parameters": [], "return_annotation": "list[Path]", "docstring": "Return tracked hand-written documentation pages in stable order.", "summary": "Return tracked hand-written documentation pages in stable order.", "implementation": "Function `_markdown_documents` calls `sorted`, `(REPOSITORY_ROOT / 'docs').rglob`, `path.read_text(encoding='utf-8').startswith`, `path.read_text`; returns `sorted((path for path in (REPOSITORY_ROOT / 'docs').rglob('*.md') if not path.read_text(encoding='utf-8').startswith('<…`.", "documented": true, "public": false, "addressable": true, "line": 92, "end_line": 101, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/mkdocs_hooks.py#L92-L101", "decorators": [], "calls": [ "sorted", "(REPOSITORY_ROOT / 'docs').rglob", "path.read_text(encoding='utf-8').startswith", "path.read_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "sorted((path for path in (REPOSITORY_ROOT / 'docs').rglob('*.md') if not path.read_text(encoding='utf-8').startswith('<…" ], "awaits": false, "yields": false }, { "name": "on_post_build", "qualname": "on_post_build", "full_name": "scripts.mkdocs_hooks.on_post_build", "kind": "function", "signature": "def on_post_build(config, **kwargs) -> None", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "None", "docstring": "Write Markdown mirrors, the API inventory, and the full LLM corpus.", "summary": "Write Markdown mirrors, the API inventory, and the full LLM corpus.", "implementation": "Function `on_post_build` calls `Path`, `build_inventory`, `build_repository_inventory`, `build_cli_inventory`.", "documented": true, "public": true, "addressable": true, "line": 104, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/scripts/mkdocs_hooks.py#L104-L225", "decorators": [], "calls": [ "Path", "build_inventory", "build_repository_inventory", "build_cli_inventory", "_source_revision", "_markdown_documents", "source_path.relative_to", "_pin_source_links", "source_path.read_text(encoding='utf-8').strip", "source_path.read_text", "mirror_path.parent.mkdir", "mirror_path.write_text", "full_parts.extend", "relative.as_posix", "render_module_page", "render_module_for_llms", "symbol_index_mirror.parent.mkdir", "symbol_index_mirror.write_text", "symbol_index_source.read_text", "render_cli_reference", "cli_mirror.parent.mkdir", "cli_mirror.write_text", "len", "sum", "module.to_dict", "(site_dir / 'api-inventory.json').write_text", "json.dumps", "(site_dir / 'source-inventory.json').write_text", "(site_dir / 'cli-inventory.json').write_text", "option.to_dict", "(site_dir / 'llms-full.txt').write_text", "'\\n'.join(full_parts).rstrip", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx", "path": "vllm_mlx/__init__.py", "page_path": "reference/api/vllm_mlx/index.md", "docstring": "vllm-mlx: Apple Silicon MLX backend for vLLM\n\nThis package provides native Apple Silicon GPU acceleration for vLLM\nusing Apple's MLX framework, mlx-lm for LLMs, and mlx-vlm for\nvision-language models.\n\nFeatures:\n- Continuous batching via vLLM-style scheduler\n- OpenAI-compatible API server\n- Support for LLM and multimodal models", "summary": "vllm-mlx: Apple Silicon MLX backend for vLLM This package provides native Apple Silicon GPU acceleration for vLLM using Apple's MLX framework, mlx-lm for LLMs, and mlx-vlm for vision-language models.", "line_count": 132, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/__init__.py#L1-L132", "members": [ "__version__", "__getattr__", "__all__" ], "symbols": [ { "name": "__getattr__", "qualname": "__getattr__", "full_name": "vllm_mlx.__getattr__", "kind": "function", "signature": "def __getattr__(name)", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Lazy load all components to avoid mlx_lm import on non-Apple platforms.", "summary": "Lazy load all components to avoid mlx_lm import on non-Apple platforms.", "implementation": "Function `__getattr__` calls `getattr`, `name.startswith`, `name.replace`, `AttributeError`; can raise `AttributeError`; has 11 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 21, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/__init__.py#L21-L90", "decorators": [], "calls": [ "getattr", "name.startswith", "name.replace", "AttributeError" ], "state_reads": [], "state_writes": [], "raises": [ "AttributeError" ], "return_expressions": [ "getattr(request, name)", "getattr(scheduler, name)", "getattr(engine_core, name)", "getattr(prefix_cache, name)", "getattr(paged_cache, name)", "getattr(mllm_cache, mllm_name)", "getattr(model_registry, name)", "MLXPlatform", "MLXWorker", "MLXModelRunner", "MLXAttentionBackend" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api", "path": "vllm_mlx/api/__init__.py", "page_path": "reference/api/vllm_mlx/api/index.md", "docstring": "API models, utilities, and tool calling support for vllm-mlx.\n\nThis module provides shared components used by the server:\n- Pydantic models for OpenAI-compatible API\n- Utility functions for text processing and model detection\n- Tool calling parsing and conversion", "summary": "API models, utilities, and tool calling support for vllm-mlx.", "line_count": 167, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/__init__.py#L1-L167", "members": [ "__all__" ], "symbols": [] }, { "name": "vllm_mlx.api.anthropic_adapter", "path": "vllm_mlx/api/anthropic_adapter.py", "page_path": "reference/api/vllm_mlx/api/anthropic_adapter.md", "docstring": "Adapter for converting between Anthropic Messages API and OpenAI Chat Completions API.\n\nHandles translation of:\n- Requests: Anthropic → OpenAI format\n- Responses: OpenAI → Anthropic format\n- Messages: Content blocks, tool calls, tool results", "summary": "Adapter for converting between Anthropic Messages API and OpenAI Chat Completions API.", "line_count": 321, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_adapter.py#L1-L321", "members": [ "anthropic_to_openai", "openai_to_anthropic", "_convert_message", "_convert_tool", "_convert_tool_choice", "_convert_stop_reason" ], "symbols": [ { "name": "anthropic_to_openai", "qualname": "anthropic_to_openai", "full_name": "vllm_mlx.api.anthropic_adapter.anthropic_to_openai", "kind": "function", "signature": "def anthropic_to_openai(request: AnthropicRequest) -> ChatCompletionRequest", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "AnthropicRequest", "default": "", "required": true, "description": "Anthropic Messages API request" } ], "return_annotation": "ChatCompletionRequest", "docstring": "Convert an Anthropic Messages API request to OpenAI Chat Completions format.\n\nHandles:\n- system field → system message\n- Content blocks → OpenAI message format\n- tool_use/tool_result → OpenAI tool_calls/tool messages\n- Anthropic tools → OpenAI tools\n\nArgs:\n request: Anthropic Messages API request\n\nReturns:\n OpenAI ChatCompletionRequest", "summary": "Convert an Anthropic Messages API request to OpenAI Chat Completions format.", "implementation": "Function `anthropic_to_openai` calls `isinstance`, `block.get`, `parts.append`, `'\\n'.join`; returns `ChatCompletionRequest(model=request.model, messages=messages, max_tokens=request.max_tokens, temperature=request.temper…`.", "documented": true, "public": true, "addressable": true, "line": 31, "end_line": 99, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_adapter.py#L31-L99", "decorators": [], "calls": [ "isinstance", "block.get", "parts.append", "'\\n'.join", "str", "re.sub", "messages.append", "Message", "_convert_message", "messages.extend", "_convert_tool", "_convert_tool_choice", "ChatCompletionRequest" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "ChatCompletionRequest(model=request.model, messages=messages, max_tokens=request.max_tokens, temperature=request.temper…" ], "awaits": false, "yields": false }, { "name": "openai_to_anthropic", "qualname": "openai_to_anthropic", "full_name": "vllm_mlx.api.anthropic_adapter.openai_to_anthropic", "kind": "function", "signature": "def openai_to_anthropic(response: ChatCompletionResponse, model: str) -> AnthropicResponse", "parameters": [ { "name": "response", "kind": "positional or keyword", "annotation": "ChatCompletionResponse", "default": "", "required": true, "description": "OpenAI ChatCompletionResponse" }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Model name for the response" } ], "return_annotation": "AnthropicResponse", "docstring": "Convert an OpenAI Chat Completions response to Anthropic Messages API format.\n\nArgs:\n response: OpenAI ChatCompletionResponse\n model: Model name for the response\n\nReturns:\n Anthropic Messages API response", "summary": "Convert an OpenAI Chat Completions response to Anthropic Messages API format.", "implementation": "Function `openai_to_anthropic` calls `content.append`, `AnthropicResponseContentBlock`, `json.loads`, `_convert_stop_reason`; returns `AnthropicResponse(model=model, content=content, stop_reason=stop_reason, usage=AnthropicUsage(input_tokens=response.usa…`.", "documented": true, "public": true, "addressable": true, "line": 102, "end_line": 162, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_adapter.py#L102-L162", "decorators": [], "calls": [ "content.append", "AnthropicResponseContentBlock", "json.loads", "_convert_stop_reason", "AnthropicResponse", "AnthropicUsage" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "AnthropicResponse(model=model, content=content, stop_reason=stop_reason, usage=AnthropicUsage(input_tokens=response.usa…" ], "awaits": false, "yields": false }, { "name": "_convert_message", "qualname": "_convert_message", "full_name": "vllm_mlx.api.anthropic_adapter._convert_message", "kind": "function", "signature": "def _convert_message(msg: AnthropicMessage) -> list[Message]", "parameters": [ { "name": "msg", "kind": "positional or keyword", "annotation": "AnthropicMessage", "default": "", "required": true, "description": "Anthropic message" } ], "return_annotation": "list[Message]", "docstring": "Convert an Anthropic message to one or more OpenAI messages.\n\nAnthropic tool_result blocks (sent as user messages) need to be\nsplit into separate OpenAI tool messages.\n\nArgs:\n msg: Anthropic message\n\nReturns:\n List of OpenAI messages", "summary": "Convert an Anthropic message to one or more OpenAI messages.", "implementation": "Function `_convert_message` calls `isinstance`, `Message`, `text_parts.append`, `tool_calls_for_assistant.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 165, "end_line": 261, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_adapter.py#L165-L261", "decorators": [], "calls": [ "isinstance", "Message", "text_parts.append", "tool_calls_for_assistant.append", "uuid.uuid4", "json.dumps", "item.get", "parts.append", "'\\n'.join", "tool_results.append", "str", "messages.append", "messages.extend" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[Message(role=msg.role, content=msg.content)]", "messages" ], "awaits": false, "yields": false }, { "name": "_convert_tool", "qualname": "_convert_tool", "full_name": "vllm_mlx.api.anthropic_adapter._convert_tool", "kind": "function", "signature": "def _convert_tool(tool: AnthropicToolDef) -> ToolDefinition", "parameters": [ { "name": "tool", "kind": "positional or keyword", "annotation": "AnthropicToolDef", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "ToolDefinition", "docstring": "Convert an Anthropic tool definition to OpenAI format.\n\nAnthropic: {\"name\": \"...\", \"description\": \"...\", \"input_schema\": {...}}\nOpenAI: {\"type\": \"function\", \"function\": {\"name\": \"...\", \"description\": \"...\", \"parameters\": {...}}}", "summary": "Convert an Anthropic tool definition to OpenAI format.", "implementation": "Function `_convert_tool` calls `ToolDefinition`; returns `ToolDefinition(type='function', function={'name': tool.name, 'description': tool.description or '', 'parameters': tool.…`.", "documented": true, "public": false, "addressable": true, "line": 264, "end_line": 278, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_adapter.py#L264-L278", "decorators": [], "calls": [ "ToolDefinition" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "ToolDefinition(type='function', function={'name': tool.name, 'description': tool.description or '', 'parameters': tool.…" ], "awaits": false, "yields": false }, { "name": "_convert_tool_choice", "qualname": "_convert_tool_choice", "full_name": "vllm_mlx.api.anthropic_adapter._convert_tool_choice", "kind": "function", "signature": "def _convert_tool_choice(tool_choice: dict) -> str | dict | None", "parameters": [ { "name": "tool_choice", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | dict | None", "docstring": "Convert Anthropic tool_choice to OpenAI format.\n\nAnthropic: {\"type\": \"auto\"} | {\"type\": \"any\"} | {\"type\": \"tool\", \"name\": \"...\"}\nOpenAI: \"auto\" | \"none\" | \"required\" | {\"type\": \"function\", \"function\": {\"name\": \"...\"}}", "summary": "Convert Anthropic tool_choice to OpenAI format.", "implementation": "Function `_convert_tool_choice` calls `tool_choice.get`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 281, "end_line": 302, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_adapter.py#L281-L302", "decorators": [], "calls": [ "tool_choice.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'auto'", "'required'", "{'type': 'function', 'function': {'name': tool_choice.get('name', '')}}", "'none'" ], "awaits": false, "yields": false }, { "name": "_convert_stop_reason", "qualname": "_convert_stop_reason", "full_name": "vllm_mlx.api.anthropic_adapter._convert_stop_reason", "kind": "function", "signature": "def _convert_stop_reason(openai_reason: str | None) -> str", "parameters": [ { "name": "openai_reason", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Convert OpenAI finish_reason to Anthropic stop_reason.\n\nOpenAI: \"stop\" | \"tool_calls\" | \"length\" | \"content_filter\"\nAnthropic: \"end_turn\" | \"tool_use\" | \"max_tokens\" | \"stop_sequence\"", "summary": "Convert OpenAI finish_reason to Anthropic stop_reason.", "implementation": "Function `_convert_stop_reason` calls `mapping.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 305, "end_line": 321, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_adapter.py#L305-L321", "decorators": [], "calls": [ "mapping.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'end_turn'", "mapping.get(openai_reason, 'end_turn')" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.anthropic_models", "path": "vllm_mlx/api/anthropic_models.py", "page_path": "reference/api/vllm_mlx/api/anthropic_models.md", "docstring": "Pydantic models for Anthropic Messages API.\n\nThese models define the request and response schemas for the\nAnthropic-compatible /v1/messages endpoint, enabling clients like\nClaude Code to communicate with vllm-mlx.", "summary": "Pydantic models for Anthropic Messages API.", "line_count": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L1-L113", "members": [ "AnthropicContentBlock", "AnthropicMessage", "AnthropicToolDef", "AnthropicRequest", "AnthropicUsage", "AnthropicResponseContentBlock", "AnthropicResponse" ], "symbols": [ { "name": "AnthropicContentBlock", "qualname": "AnthropicContentBlock", "full_name": "vllm_mlx.api.anthropic_models.AnthropicContentBlock", "kind": "class", "signature": "class AnthropicContentBlock(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "text", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "name", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "input", "kind": "field", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tool_use_id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "content", "kind": "field", "annotation": "str | list | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "is_error", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "source", "kind": "field", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AnthropicContentBlock", "docstring": "A content block in an Anthropic message.", "summary": "A content block in an Anthropic message.", "implementation": "Class `AnthropicContentBlock` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 20, "end_line": 35, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L20-L35", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AnthropicMessage", "qualname": "AnthropicMessage", "full_name": "vllm_mlx.api.anthropic_models.AnthropicMessage", "kind": "class", "signature": "class AnthropicMessage(BaseModel)", "parameters": [ { "name": "role", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "content", "kind": "field", "annotation": "str | list[AnthropicContentBlock]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "AnthropicMessage", "docstring": "A message in an Anthropic conversation.", "summary": "A message in an Anthropic conversation.", "implementation": "Class `AnthropicMessage` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 38, "end_line": 42, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L38-L42", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AnthropicToolDef", "qualname": "AnthropicToolDef", "full_name": "vllm_mlx.api.anthropic_models.AnthropicToolDef", "kind": "class", "signature": "class AnthropicToolDef(BaseModel)", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "input_schema", "kind": "field", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AnthropicToolDef", "docstring": "Definition of a tool in Anthropic format.", "summary": "Definition of a tool in Anthropic format.", "implementation": "Class `AnthropicToolDef` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L45-L50", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AnthropicRequest", "qualname": "AnthropicRequest", "full_name": "vllm_mlx.api.anthropic_models.AnthropicRequest", "kind": "class", "signature": "class AnthropicRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "messages", "kind": "field", "annotation": "list[AnthropicMessage]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "system", "kind": "field", "annotation": "str | list[dict] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_tokens", "kind": "field", "annotation": "int", "default": "Field(gt=0)", "required": false, "description": "Optional constructor field; defaults to `Field(gt=0)`." }, { "name": "temperature", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "top_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "stream", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "stop_sequences", "kind": "field", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tools", "kind": "field", "annotation": "list[AnthropicToolDef] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tool_choice", "kind": "field", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "metadata", "kind": "field", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "top_k", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "response_format", "kind": "field", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "chat_template_kwargs", "kind": "field", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AnthropicRequest", "docstring": "Request for Anthropic Messages API.", "summary": "Request for Anthropic Messages API.", "implementation": "Class `AnthropicRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 53, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L53-L73", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AnthropicUsage", "qualname": "AnthropicUsage", "full_name": "vllm_mlx.api.anthropic_models.AnthropicUsage", "kind": "class", "signature": "class AnthropicUsage(BaseModel)", "parameters": [ { "name": "input_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "output_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "cache_creation_input_tokens", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "cache_read_input_tokens", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AnthropicUsage", "docstring": "Token usage for Anthropic response.", "summary": "Token usage for Anthropic response.", "implementation": "Class `AnthropicUsage` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 81, "end_line": 87, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L81-L87", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AnthropicResponseContentBlock", "qualname": "AnthropicResponseContentBlock", "full_name": "vllm_mlx.api.anthropic_models.AnthropicResponseContentBlock", "kind": "class", "signature": "class AnthropicResponseContentBlock(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "text", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "thinking", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "name", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "input", "kind": "field", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AnthropicResponseContentBlock", "docstring": "A content block in the Anthropic response.", "summary": "A content block in the Anthropic response.", "implementation": "Class `AnthropicResponseContentBlock` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 90, "end_line": 100, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L90-L100", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AnthropicResponse", "qualname": "AnthropicResponse", "full_name": "vllm_mlx.api.anthropic_models.AnthropicResponse", "kind": "class", "signature": "class AnthropicResponse(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str", "default": "Field(default_factory=lambda: f'msg_{uuid.uuid4().hex[:24]}')", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: f'msg_{uuid.uuid4().hex[:24]}')`." }, { "name": "type", "kind": "field", "annotation": "str", "default": "'message'", "required": false, "description": "Optional constructor field; defaults to `'message'`." }, { "name": "role", "kind": "field", "annotation": "str", "default": "'assistant'", "required": false, "description": "Optional constructor field; defaults to `'assistant'`." }, { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "content", "kind": "field", "annotation": "list[AnthropicResponseContentBlock]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "stop_reason", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "stop_sequence", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "usage", "kind": "field", "annotation": "AnthropicUsage", "default": "Field(default_factory=AnthropicUsage)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=AnthropicUsage)`." } ], "return_annotation": "AnthropicResponse", "docstring": "Response for Anthropic Messages API.", "summary": "Response for Anthropic Messages API.", "implementation": "Class `AnthropicResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 103, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/anthropic_models.py#L103-L113", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.harmony_tools", "path": "vllm_mlx/api/harmony_tools.py", "page_path": "reference/api/vllm_mlx/api/harmony_tools.md", "docstring": "TypeScript-style tool definition converter for Harmony/GPT-OSS models.\n\nHarmony models expect tool definitions in TypeScript namespace format:\n\n namespace functions {\n // Get weather for a location\n type get_weather = (_: {\n location: string,\n unit?: \"celsius\" | \"fahrenheit\"\n }) => any;\n }\n\nThis module converts OpenAI JSON Schema tool definitions to that format.", "summary": "TypeScript-style tool definition converter for Harmony/GPT-OSS models.", "line_count": 109, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/harmony_tools.py#L1-L109", "members": [ "_TYPE_MAP", "_convert_type", "convert_tools_to_typescript" ], "symbols": [ { "name": "_convert_type", "qualname": "_convert_type", "full_name": "vllm_mlx.api.harmony_tools._convert_type", "kind": "function", "signature": "def _convert_type(prop: dict[str, Any]) -> str", "parameters": [ { "name": "prop", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "JSON Schema property definition." } ], "return_annotation": "str", "docstring": "Convert a JSON Schema property to a TypeScript type string.\n\nArgs:\n prop: JSON Schema property definition.\n\nReturns:\n TypeScript type string.", "summary": "Convert a JSON Schema property to a TypeScript type string.", "implementation": "Function `_convert_type` calls `' | '.join`, `prop.get`, `_convert_type`, `_TYPE_MAP.get`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 31, "end_line": 54, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/harmony_tools.py#L31-L54", "decorators": [], "calls": [ "' | '.join", "prop.get", "_convert_type", "_TYPE_MAP.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "' | '.join(literals)", "f'Array<{item_type}>'", "_TYPE_MAP.get(schema_type, 'any')" ], "awaits": false, "yields": false }, { "name": "convert_tools_to_typescript", "qualname": "convert_tools_to_typescript", "full_name": "vllm_mlx.api.harmony_tools.convert_tools_to_typescript", "kind": "function", "signature": "def convert_tools_to_typescript(tools: list[dict[str, Any]] | None) -> str | None", "parameters": [ { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict[str, Any]] | None", "default": "", "required": true, "description": "List of tool definitions in OpenAI format, e.g.: [{\"type\": \"function\", \"function\": {\"name\": \"...\", ...}}]" } ], "return_annotation": "str | None", "docstring": "Convert OpenAI JSON Schema tool definitions to TypeScript namespace format.\n\nArgs:\n tools: List of tool definitions in OpenAI format, e.g.:\n [{\"type\": \"function\", \"function\": {\"name\": \"...\", ...}}]\n\nReturns:\n TypeScript namespace string, or None if no tools.", "summary": "Convert OpenAI JSON Schema tool definitions to TypeScript namespace format.", "implementation": "Function `convert_tools_to_typescript` calls `tool.get`, `func.get`, `parameters.get`, `set`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 57, "end_line": 109, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/harmony_tools.py#L57-L109", "decorators": [], "calls": [ "tool.get", "func.get", "parameters.get", "set", "properties.items", "_convert_type", "params.append", "lines.append", "'\\n'.join", "functions.append", "'\\n\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "f'namespace functions {{\\n{body}\\n}}'" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.models", "path": "vllm_mlx/api/models.py", "page_path": "reference/api/vllm_mlx/api/models.md", "docstring": "Pydantic models for OpenAI-compatible API.\n\nThese models define the request and response schemas for:\n- Chat completions\n- Text completions\n- Tool calling\n- MCP (Model Context Protocol) integration", "summary": "Pydantic models for OpenAI-compatible API.", "line_count": 581, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L1-L581", "members": [ "ImageUrl", "VideoUrl", "AudioUrl", "ContentPart", "Message", "_OPENAI_FUNCTION_NAME_RE", "FunctionCall", "ToolCall", "ToolDefinition", "ResponseFormatJsonSchema", "ResponseFormat", "StreamOptions", "ChatCompletionRequest", "AssistantMessage", "ChatCompletionChoice", "Usage", "GenerationMetadata", "ChatCompletionResponse", "CompletionRequest", "CompletionChoice", "CompletionResponse", "ModelInfo", "ModelsResponse", "MCPToolInfo", "MCPToolsResponse", "MCPServerInfo", "MCPServersResponse", "MCPExecuteRequest", "MCPExecuteResponse", "AudioTranscriptionRequest", "AudioTranscriptionResponse", "AudioSpeechRequest", "AudioSeparationRequest", "EmbeddingRequest", "EmbeddingData", "EmbeddingUsage", "EmbeddingResponse", "RerankRequest", "RerankResult", "RerankUsage", "RerankResponse", "ChatCompletionChunkDelta", "ChatCompletionChunkChoice", "ChatCompletionChunk" ], "symbols": [ { "name": "ImageUrl", "qualname": "ImageUrl", "full_name": "vllm_mlx.api.models.ImageUrl", "kind": "class", "signature": "class ImageUrl(BaseModel)", "parameters": [ { "name": "url", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "detail", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ImageUrl", "docstring": "Image URL with optional detail level.", "summary": "Image URL with optional detail level.", "implementation": "Class `ImageUrl` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 24, "end_line": 28, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L24-L28", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "VideoUrl", "qualname": "VideoUrl", "full_name": "vllm_mlx.api.models.VideoUrl", "kind": "class", "signature": "class VideoUrl(BaseModel)", "parameters": [ { "name": "url", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "VideoUrl", "docstring": "Video URL.", "summary": "Video URL.", "implementation": "Class `VideoUrl` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 31, "end_line": 34, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L31-L34", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AudioUrl", "qualname": "AudioUrl", "full_name": "vllm_mlx.api.models.AudioUrl", "kind": "class", "signature": "class AudioUrl(BaseModel)", "parameters": [ { "name": "url", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "AudioUrl", "docstring": "Audio URL for audio content.", "summary": "Audio URL for audio content.", "implementation": "Class `AudioUrl` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 37, "end_line": 40, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L37-L40", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ContentPart", "qualname": "ContentPart", "full_name": "vllm_mlx.api.models.ContentPart", "kind": "class", "signature": "class ContentPart(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "text", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "image_url", "kind": "field", "annotation": "ImageUrl | dict | str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "video", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "video_url", "kind": "field", "annotation": "VideoUrl | dict | str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "audio_url", "kind": "field", "annotation": "AudioUrl | dict | str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ContentPart", "docstring": "A part of a multimodal message content.\n\nSupports:\n- text: Plain text content\n- image_url: Image from URL or base64\n- video: Video from local path\n- video_url: Video from URL or base64\n- audio_url: Audio from URL or base64", "summary": "A part of a multimodal message content.", "implementation": "Class `ContentPart` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 43, "end_line": 60, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L43-L60", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "Message", "qualname": "Message", "full_name": "vllm_mlx.api.models.Message", "kind": "class", "signature": "class Message(BaseModel)", "parameters": [ { "name": "role", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "content", "kind": "field", "annotation": "str | list[ContentPart] | list[dict] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tool_calls", "kind": "field", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tool_call_id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "Message", "docstring": "A message in a chat conversation.\n\nSupports:\n- Simple text messages (role + content string)\n- Multimodal messages (role + content list with text/images/videos)\n- Tool call messages (assistant with tool_calls)\n- Tool response messages (role=\"tool\" with tool_call_id)", "summary": "A message in a chat conversation.", "implementation": "Class `Message` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 68, "end_line": 84, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L68-L84", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "FunctionCall", "qualname": "FunctionCall", "full_name": "vllm_mlx.api.models.FunctionCall", "kind": "class", "signature": "class FunctionCall(BaseModel)", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "arguments", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "FunctionCall", "docstring": "A function call with name and arguments.", "summary": "A function call with name and arguments.", "implementation": "Class `FunctionCall` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 95, "end_line": 99, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L95-L99", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ToolCall", "qualname": "ToolCall", "full_name": "vllm_mlx.api.models.ToolCall", "kind": "class", "signature": "class ToolCall(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "type", "kind": "field", "annotation": "str", "default": "'function'", "required": false, "description": "Optional constructor field; defaults to `'function'`." }, { "name": "function", "kind": "field", "annotation": "FunctionCall", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ToolCall", "docstring": "A tool call from the model.", "summary": "A tool call from the model.", "implementation": "Class `ToolCall` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 102, "end_line": 107, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L102-L107", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ToolDefinition", "qualname": "ToolDefinition", "full_name": "vllm_mlx.api.models.ToolDefinition", "kind": "class", "signature": "class ToolDefinition(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "str", "default": "'function'", "required": false, "description": "Optional constructor field; defaults to `'function'`." }, { "name": "function", "kind": "field", "annotation": "dict", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ToolDefinition", "docstring": "Definition of a tool that can be called by the model.", "summary": "Definition of a tool that can be called by the model.", "implementation": "Class `ToolDefinition` derives from `BaseModel` and declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 110, "end_line": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L110-L123", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_validate_openai_function_name", "qualname": "ToolDefinition._validate_openai_function_name", "full_name": "vllm_mlx.api.models.ToolDefinition._validate_openai_function_name", "kind": "method", "signature": "def _validate_openai_function_name(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `ToolDefinition._validate_openai_function_name` calls `self.function.get`, `isinstance`, `_OPENAI_FUNCTION_NAME_RE.fullmatch`, `ValueError`; can raise `ValueError`; returns `self`.", "implementation": "Method `ToolDefinition._validate_openai_function_name` calls `self.function.get`, `isinstance`, `_OPENAI_FUNCTION_NAME_RE.fullmatch`, `ValueError`; can raise `ValueError`; returns `self`.", "documented": false, "public": false, "addressable": true, "line": 117, "end_line": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L117-L123", "decorators": [ "model_validator(mode='after')" ], "calls": [ "self.function.get", "isinstance", "_OPENAI_FUNCTION_NAME_RE.fullmatch", "ValueError" ], "state_reads": [ "self.type", "self.function.get", "self.function" ], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "self" ], "awaits": false, "yields": false }, { "name": "ResponseFormatJsonSchema", "qualname": "ResponseFormatJsonSchema", "full_name": "vllm_mlx.api.models.ResponseFormatJsonSchema", "kind": "class", "signature": "class ResponseFormatJsonSchema(BaseModel)", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "schema_", "kind": "field", "annotation": "dict", "default": "Field(alias='schema')", "required": false, "description": "Optional constructor field; defaults to `Field(alias='schema')`." }, { "name": "strict", "kind": "field", "annotation": "bool | None", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "ResponseFormatJsonSchema", "docstring": "JSON Schema definition for structured output.", "summary": "JSON Schema definition for structured output.", "implementation": "Class `ResponseFormatJsonSchema` derives from `BaseModel` and declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 131, "end_line": 142, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L131-L142", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "Config", "qualname": "ResponseFormatJsonSchema.Config", "full_name": "vllm_mlx.api.models.ResponseFormatJsonSchema.Config", "kind": "class", "signature": "class Config", "parameters": [], "return_annotation": "Config", "docstring": "Allow callers to populate the aliased ``schema`` field by name.", "summary": "Allow callers to populate the aliased ``schema`` field by name.", "implementation": "Class `ResponseFormatJsonSchema.Config` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 139, "end_line": 142, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L139-L142", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseFormat", "qualname": "ResponseFormat", "full_name": "vllm_mlx.api.models.ResponseFormat", "kind": "class", "signature": "class ResponseFormat(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "str", "default": "'text'", "required": false, "description": "Optional constructor field; defaults to `'text'`." }, { "name": "json_schema", "kind": "field", "annotation": "ResponseFormatJsonSchema | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ResponseFormat", "docstring": "Response format specification for structured output.\n\nSupports:\n- \"text\": Default text output (no structure enforcement)\n- \"json_object\": Forces valid JSON output\n- \"json_schema\": Forces JSON matching a specific schema", "summary": "Response format specification for structured output.", "implementation": "Class `ResponseFormat` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 145, "end_line": 156, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L145-L156", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "StreamOptions", "qualname": "StreamOptions", "full_name": "vllm_mlx.api.models.StreamOptions", "kind": "class", "signature": "class StreamOptions(BaseModel)", "parameters": [ { "name": "include_usage", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "StreamOptions", "docstring": "Options for streaming responses.", "summary": "Options for streaming responses.", "implementation": "Class `StreamOptions` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 164, "end_line": 167, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L164-L167", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ChatCompletionRequest", "qualname": "ChatCompletionRequest", "full_name": "vllm_mlx.api.models.ChatCompletionRequest", "kind": "class", "signature": "class ChatCompletionRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "messages", "kind": "field", "annotation": "list[Message]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "temperature", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "top_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "top_k", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "min_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "presence_penalty", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_tokens", "kind": "field", "annotation": "int | None", "default": "Field(default=None, gt=0)", "required": false, "description": "Optional constructor field; defaults to `Field(default=None, gt=0)`." }, { "name": "stream", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "stream_options", "kind": "field", "annotation": "StreamOptions | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "stop", "kind": "field", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tools", "kind": "field", "annotation": "list[ToolDefinition] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tool_choice", "kind": "field", "annotation": "str | dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "response_format", "kind": "field", "annotation": "ResponseFormat | dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "logit_bias", "kind": "field", "annotation": "dict[str, float] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "chat_template_kwargs", "kind": "field", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "video_fps", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "video_max_frames", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "repetition_penalty", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "timeout", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_keep_pct", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_backbone_pct", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "enable_thinking", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "mllm_draft", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "thinking_token_budget", "kind": "field", "annotation": "int | None", "default": "Field(default=None, gt=0)", "required": false, "description": "Optional constructor field; defaults to `Field(default=None, gt=0)`." } ], "return_annotation": "ChatCompletionRequest", "docstring": "Request for chat completion.", "summary": "Request for chat completion.", "implementation": "Class `ChatCompletionRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 170, "end_line": 216, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L170-L216", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AssistantMessage", "qualname": "AssistantMessage", "full_name": "vllm_mlx.api.models.AssistantMessage", "kind": "class", "signature": "class AssistantMessage(BaseModel)", "parameters": [ { "name": "role", "kind": "field", "annotation": "str", "default": "'assistant'", "required": false, "description": "Optional constructor field; defaults to `'assistant'`." }, { "name": "content", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "reasoning_content", "kind": "field", "annotation": "str | None", "default": "Field(default=None, validation_alias=AliasChoices('reasoning_content', 'reasoning'))", "required": false, "description": "Optional constructor field; defaults to `Field(default=None, validation_alias=AliasChoices('reasoning_content', 'reasoning'))`." }, { "name": "tool_calls", "kind": "field", "annotation": "list[ToolCall] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AssistantMessage", "docstring": "Response message from the assistant.", "summary": "Response message from the assistant.", "implementation": "Class `AssistantMessage` derives from `BaseModel` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 219, "end_line": 248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L219-L248", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reasoning", "qualname": "AssistantMessage.reasoning", "full_name": "vllm_mlx.api.models.AssistantMessage.reasoning", "kind": "method", "signature": "def reasoning(self) -> str | None", "parameters": [], "return_annotation": "str | None", "docstring": "Return reasoning content through the legacy compatibility alias.", "summary": "Return reasoning content through the legacy compatibility alias.", "implementation": "Method `AssistantMessage.reasoning` returns `self.reasoning_content`.", "documented": true, "public": true, "addressable": true, "line": 231, "end_line": 234, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L231-L234", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.reasoning_content" ], "state_writes": [], "raises": [], "return_expressions": [ "self.reasoning_content" ], "awaits": false, "yields": false }, { "name": "_serialize", "qualname": "AssistantMessage._serialize", "full_name": "vllm_mlx.api.models.AssistantMessage._serialize", "kind": "method", "signature": "def _serialize(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Serialize with OpenAI-compatible schema.\n\n- ``tool_calls`` and ``reasoning_content`` are omitted when None.\n- ``content`` is always included (even as null) per OpenAI spec.", "summary": "Serialize with OpenAI-compatible schema.", "implementation": "Method `AssistantMessage._serialize` calls `tc.model_dump`; returns `d`.", "documented": true, "public": false, "addressable": true, "line": 237, "end_line": 248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L237-L248", "decorators": [ "model_serializer" ], "calls": [ "tc.model_dump" ], "state_reads": [ "self.role", "self.content", "self.reasoning_content", "self.tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "d" ], "awaits": false, "yields": false }, { "name": "ChatCompletionChoice", "qualname": "ChatCompletionChoice", "full_name": "vllm_mlx.api.models.ChatCompletionChoice", "kind": "class", "signature": "class ChatCompletionChoice(BaseModel)", "parameters": [ { "name": "index", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "message", "kind": "field", "annotation": "AssistantMessage", "default": "", "required": true, "description": "Required constructor field." }, { "name": "finish_reason", "kind": "field", "annotation": "str | None", "default": "'stop'", "required": false, "description": "Optional constructor field; defaults to `'stop'`." } ], "return_annotation": "ChatCompletionChoice", "docstring": "A single choice in chat completion response.", "summary": "A single choice in chat completion response.", "implementation": "Class `ChatCompletionChoice` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 251, "end_line": 256, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L251-L256", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "Usage", "qualname": "Usage", "full_name": "vllm_mlx.api.models.Usage", "kind": "class", "signature": "class Usage(BaseModel)", "parameters": [ { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "completion_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "total_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "Usage", "docstring": "Token usage statistics.", "summary": "Token usage statistics.", "implementation": "Class `Usage` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 259, "end_line": 264, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L259-L264", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "GenerationMetadata", "qualname": "GenerationMetadata", "full_name": "vllm_mlx.api.models.GenerationMetadata", "kind": "class", "signature": "class GenerationMetadata(BaseModel)", "parameters": [ { "name": "no_final_content_watchdog_tokens", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "no_final_content_watchdog_enforced", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "GenerationMetadata", "docstring": "Optional generation diagnostics emitted for feature-bearing requests.", "summary": "Optional generation diagnostics emitted for feature-bearing requests.", "implementation": "Class `GenerationMetadata` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 267, "end_line": 271, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L267-L271", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ChatCompletionResponse", "qualname": "ChatCompletionResponse", "full_name": "vllm_mlx.api.models.ChatCompletionResponse", "kind": "class", "signature": "class ChatCompletionResponse(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str", "default": "Field(default_factory=lambda: f'chatcmpl-{uuid.uuid4().hex[:8]}')", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: f'chatcmpl-{uuid.uuid4().hex[:8]}')`." }, { "name": "object", "kind": "field", "annotation": "str", "default": "'chat.completion'", "required": false, "description": "Optional constructor field; defaults to `'chat.completion'`." }, { "name": "created", "kind": "field", "annotation": "int", "default": "Field(default_factory=lambda: int(time.time()))", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: int(time.time()))`." }, { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "choices", "kind": "field", "annotation": "list[ChatCompletionChoice]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "usage", "kind": "field", "annotation": "Usage", "default": "Field(default_factory=Usage)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=Usage)`." }, { "name": "generation_metadata", "kind": "field", "annotation": "GenerationMetadata | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ChatCompletionResponse", "docstring": "Response for chat completion.", "summary": "Response for chat completion.", "implementation": "Class `ChatCompletionResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 274, "end_line": 283, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L274-L283", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "CompletionRequest", "qualname": "CompletionRequest", "full_name": "vllm_mlx.api.models.CompletionRequest", "kind": "class", "signature": "class CompletionRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt", "kind": "field", "annotation": "str | list[str]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "temperature", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "top_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "top_k", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "min_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "presence_penalty", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_tokens", "kind": "field", "annotation": "int | None", "default": "Field(default=None, gt=0)", "required": false, "description": "Optional constructor field; defaults to `Field(default=None, gt=0)`." }, { "name": "stream", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "stop", "kind": "field", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "repetition_penalty", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "timeout", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_keep_pct", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_backbone_pct", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "CompletionRequest", "docstring": "Request for text completion.", "summary": "Request for text completion.", "implementation": "Class `CompletionRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 291, "end_line": 313, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L291-L313", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "CompletionChoice", "qualname": "CompletionChoice", "full_name": "vllm_mlx.api.models.CompletionChoice", "kind": "class", "signature": "class CompletionChoice(BaseModel)", "parameters": [ { "name": "index", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "finish_reason", "kind": "field", "annotation": "str | None", "default": "'stop'", "required": false, "description": "Optional constructor field; defaults to `'stop'`." } ], "return_annotation": "CompletionChoice", "docstring": "A single choice in text completion response.", "summary": "A single choice in text completion response.", "implementation": "Class `CompletionChoice` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 316, "end_line": 321, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L316-L321", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "CompletionResponse", "qualname": "CompletionResponse", "full_name": "vllm_mlx.api.models.CompletionResponse", "kind": "class", "signature": "class CompletionResponse(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str", "default": "Field(default_factory=lambda: f'cmpl-{uuid.uuid4().hex[:8]}')", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: f'cmpl-{uuid.uuid4().hex[:8]}')`." }, { "name": "object", "kind": "field", "annotation": "str", "default": "'text_completion'", "required": false, "description": "Optional constructor field; defaults to `'text_completion'`." }, { "name": "created", "kind": "field", "annotation": "int", "default": "Field(default_factory=lambda: int(time.time()))", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: int(time.time()))`." }, { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "choices", "kind": "field", "annotation": "list[CompletionChoice]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "usage", "kind": "field", "annotation": "Usage", "default": "Field(default_factory=Usage)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=Usage)`." } ], "return_annotation": "CompletionResponse", "docstring": "Response for text completion.", "summary": "Response for text completion.", "implementation": "Class `CompletionResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 324, "end_line": 332, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L324-L332", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ModelInfo", "qualname": "ModelInfo", "full_name": "vllm_mlx.api.models.ModelInfo", "kind": "class", "signature": "class ModelInfo(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "object", "kind": "field", "annotation": "str", "default": "'model'", "required": false, "description": "Optional constructor field; defaults to `'model'`." }, { "name": "created", "kind": "field", "annotation": "int", "default": "Field(default_factory=lambda: int(time.time()))", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: int(time.time()))`." }, { "name": "owned_by", "kind": "field", "annotation": "str", "default": "'vllm-mlx'", "required": false, "description": "Optional constructor field; defaults to `'vllm-mlx'`." } ], "return_annotation": "ModelInfo", "docstring": "Information about an available model.", "summary": "Information about an available model.", "implementation": "Class `ModelInfo` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 340, "end_line": 346, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L340-L346", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ModelsResponse", "qualname": "ModelsResponse", "full_name": "vllm_mlx.api.models.ModelsResponse", "kind": "class", "signature": "class ModelsResponse(BaseModel)", "parameters": [ { "name": "object", "kind": "field", "annotation": "str", "default": "'list'", "required": false, "description": "Optional constructor field; defaults to `'list'`." }, { "name": "data", "kind": "field", "annotation": "list[ModelInfo]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ModelsResponse", "docstring": "Response for listing models.", "summary": "Response for listing models.", "implementation": "Class `ModelsResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 349, "end_line": 353, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L349-L353", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPToolInfo", "qualname": "MCPToolInfo", "full_name": "vllm_mlx.api.models.MCPToolInfo", "kind": "class", "signature": "class MCPToolInfo(BaseModel)", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "server", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "parameters", "kind": "field", "annotation": "dict", "default": "Field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=dict)`." } ], "return_annotation": "MCPToolInfo", "docstring": "Information about an MCP tool.", "summary": "Information about an MCP tool.", "implementation": "Class `MCPToolInfo` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 361, "end_line": 367, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L361-L367", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPToolsResponse", "qualname": "MCPToolsResponse", "full_name": "vllm_mlx.api.models.MCPToolsResponse", "kind": "class", "signature": "class MCPToolsResponse(BaseModel)", "parameters": [ { "name": "tools", "kind": "field", "annotation": "list[MCPToolInfo]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "count", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "MCPToolsResponse", "docstring": "Response for listing MCP tools.", "summary": "Response for listing MCP tools.", "implementation": "Class `MCPToolsResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 370, "end_line": 374, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L370-L374", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPServerInfo", "qualname": "MCPServerInfo", "full_name": "vllm_mlx.api.models.MCPServerInfo", "kind": "class", "signature": "class MCPServerInfo(BaseModel)", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "state", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "transport", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tools_count", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "error", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MCPServerInfo", "docstring": "Information about an MCP server.", "summary": "Information about an MCP server.", "implementation": "Class `MCPServerInfo` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 377, "end_line": 384, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L377-L384", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPServersResponse", "qualname": "MCPServersResponse", "full_name": "vllm_mlx.api.models.MCPServersResponse", "kind": "class", "signature": "class MCPServersResponse(BaseModel)", "parameters": [ { "name": "servers", "kind": "field", "annotation": "list[MCPServerInfo]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "MCPServersResponse", "docstring": "Response for listing MCP servers.", "summary": "Response for listing MCP servers.", "implementation": "Class `MCPServersResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 387, "end_line": 390, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L387-L390", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPExecuteRequest", "qualname": "MCPExecuteRequest", "full_name": "vllm_mlx.api.models.MCPExecuteRequest", "kind": "class", "signature": "class MCPExecuteRequest(BaseModel)", "parameters": [ { "name": "tool_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "arguments", "kind": "field", "annotation": "dict", "default": "Field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=dict)`." } ], "return_annotation": "MCPExecuteRequest", "docstring": "Request to execute an MCP tool.", "summary": "Request to execute an MCP tool.", "implementation": "Class `MCPExecuteRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 393, "end_line": 397, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L393-L397", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPExecuteResponse", "qualname": "MCPExecuteResponse", "full_name": "vllm_mlx.api.models.MCPExecuteResponse", "kind": "class", "signature": "class MCPExecuteResponse(BaseModel)", "parameters": [ { "name": "tool_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "content", "kind": "field", "annotation": "str | list | dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "is_error", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "error_message", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MCPExecuteResponse", "docstring": "Response from executing an MCP tool.", "summary": "Response from executing an MCP tool.", "implementation": "Class `MCPExecuteResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 400, "end_line": 406, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L400-L406", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AudioTranscriptionRequest", "qualname": "AudioTranscriptionRequest", "full_name": "vllm_mlx.api.models.AudioTranscriptionRequest", "kind": "class", "signature": "class AudioTranscriptionRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "'whisper-large-v3'", "required": false, "description": "Optional constructor field; defaults to `'whisper-large-v3'`." }, { "name": "language", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "response_format", "kind": "field", "annotation": "str", "default": "'json'", "required": false, "description": "Optional constructor field; defaults to `'json'`." }, { "name": "temperature", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "timestamp_granularities", "kind": "field", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AudioTranscriptionRequest", "docstring": "Request for audio transcription (STT).", "summary": "Request for audio transcription (STT).", "implementation": "Class `AudioTranscriptionRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 414, "end_line": 421, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L414-L421", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AudioTranscriptionResponse", "qualname": "AudioTranscriptionResponse", "full_name": "vllm_mlx.api.models.AudioTranscriptionResponse", "kind": "class", "signature": "class AudioTranscriptionResponse(BaseModel)", "parameters": [ { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "language", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "duration", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "segments", "kind": "field", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "AudioTranscriptionResponse", "docstring": "Response from audio transcription.", "summary": "Response from audio transcription.", "implementation": "Class `AudioTranscriptionResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 424, "end_line": 430, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L424-L430", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AudioSpeechRequest", "qualname": "AudioSpeechRequest", "full_name": "vllm_mlx.api.models.AudioSpeechRequest", "kind": "class", "signature": "class AudioSpeechRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "'kokoro'", "required": false, "description": "Optional constructor field; defaults to `'kokoro'`." }, { "name": "input", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "voice", "kind": "field", "annotation": "str", "default": "'af_heart'", "required": false, "description": "Optional constructor field; defaults to `'af_heart'`." }, { "name": "speed", "kind": "field", "annotation": "float", "default": "1.0", "required": false, "description": "Optional constructor field; defaults to `1.0`." }, { "name": "response_format", "kind": "field", "annotation": "str", "default": "'wav'", "required": false, "description": "Optional constructor field; defaults to `'wav'`." } ], "return_annotation": "AudioSpeechRequest", "docstring": "Request for text-to-speech.", "summary": "Request for text-to-speech.", "implementation": "Class `AudioSpeechRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 433, "end_line": 440, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L433-L440", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AudioSeparationRequest", "qualname": "AudioSeparationRequest", "full_name": "vllm_mlx.api.models.AudioSeparationRequest", "kind": "class", "signature": "class AudioSeparationRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "'htdemucs'", "required": false, "description": "Optional constructor field; defaults to `'htdemucs'`." }, { "name": "stems", "kind": "field", "annotation": "list[str]", "default": "Field(default_factory=lambda: ['vocals', 'accompaniment'])", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: ['vocals', 'accompaniment'])`." } ], "return_annotation": "AudioSeparationRequest", "docstring": "Request for audio source separation.", "summary": "Request for audio source separation.", "implementation": "Class `AudioSeparationRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 443, "end_line": 447, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L443-L447", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "EmbeddingRequest", "qualname": "EmbeddingRequest", "full_name": "vllm_mlx.api.models.EmbeddingRequest", "kind": "class", "signature": "class EmbeddingRequest(BaseModel)", "parameters": [ { "name": "input", "kind": "field", "annotation": "str | list[str]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "encoding_format", "kind": "field", "annotation": "str | None", "default": "'float'", "required": false, "description": "Optional constructor field; defaults to `'float'`." } ], "return_annotation": "EmbeddingRequest", "docstring": "Request for text embeddings (OpenAI compatible).", "summary": "Request for text embeddings (OpenAI compatible).", "implementation": "Class `EmbeddingRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 455, "end_line": 460, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L455-L460", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "EmbeddingData", "qualname": "EmbeddingData", "full_name": "vllm_mlx.api.models.EmbeddingData", "kind": "class", "signature": "class EmbeddingData(BaseModel)", "parameters": [ { "name": "object", "kind": "field", "annotation": "str", "default": "'embedding'", "required": false, "description": "Optional constructor field; defaults to `'embedding'`." }, { "name": "index", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "embedding", "kind": "field", "annotation": "list[float]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "EmbeddingData", "docstring": "A single embedding result.", "summary": "A single embedding result.", "implementation": "Class `EmbeddingData` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 463, "end_line": 468, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L463-L468", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "EmbeddingUsage", "qualname": "EmbeddingUsage", "full_name": "vllm_mlx.api.models.EmbeddingUsage", "kind": "class", "signature": "class EmbeddingUsage(BaseModel)", "parameters": [ { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "total_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "EmbeddingUsage", "docstring": "Token usage for embedding requests.", "summary": "Token usage for embedding requests.", "implementation": "Class `EmbeddingUsage` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 471, "end_line": 475, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L471-L475", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "EmbeddingResponse", "qualname": "EmbeddingResponse", "full_name": "vllm_mlx.api.models.EmbeddingResponse", "kind": "class", "signature": "class EmbeddingResponse(BaseModel)", "parameters": [ { "name": "object", "kind": "field", "annotation": "str", "default": "'list'", "required": false, "description": "Optional constructor field; defaults to `'list'`." }, { "name": "data", "kind": "field", "annotation": "list[EmbeddingData]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "usage", "kind": "field", "annotation": "EmbeddingUsage", "default": "Field(default_factory=EmbeddingUsage)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=EmbeddingUsage)`." } ], "return_annotation": "EmbeddingResponse", "docstring": "Response for embeddings endpoint (OpenAI compatible).", "summary": "Response for embeddings endpoint (OpenAI compatible).", "implementation": "Class `EmbeddingResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 478, "end_line": 484, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L478-L484", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RerankRequest", "qualname": "RerankRequest", "full_name": "vllm_mlx.api.models.RerankRequest", "kind": "class", "signature": "class RerankRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "query", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "documents", "kind": "field", "annotation": "list[str | dict]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "top_n", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "return_documents", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." } ], "return_annotation": "RerankRequest", "docstring": "Request for reranking documents against a query (Jina/Cohere convention).", "summary": "Request for reranking documents against a query (Jina/Cohere convention).", "implementation": "Class `RerankRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 492, "end_line": 499, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L492-L499", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RerankResult", "qualname": "RerankResult", "full_name": "vllm_mlx.api.models.RerankResult", "kind": "class", "signature": "class RerankResult(BaseModel)", "parameters": [ { "name": "index", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "relevance_score", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "document", "kind": "field", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "RerankResult", "docstring": "A single reranked document result.", "summary": "A single reranked document result.", "implementation": "Class `RerankResult` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 502, "end_line": 507, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L502-L507", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RerankUsage", "qualname": "RerankUsage", "full_name": "vllm_mlx.api.models.RerankUsage", "kind": "class", "signature": "class RerankUsage(BaseModel)", "parameters": [ { "name": "total_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "RerankUsage", "docstring": "Token usage for rerank requests.", "summary": "Token usage for rerank requests.", "implementation": "Class `RerankUsage` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 510, "end_line": 513, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L510-L513", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RerankResponse", "qualname": "RerankResponse", "full_name": "vllm_mlx.api.models.RerankResponse", "kind": "class", "signature": "class RerankResponse(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "results", "kind": "field", "annotation": "list[RerankResult]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "usage", "kind": "field", "annotation": "RerankUsage", "default": "Field(default_factory=RerankUsage)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=RerankUsage)`." } ], "return_annotation": "RerankResponse", "docstring": "Response for reranking endpoint (Jina/Cohere convention).", "summary": "Response for reranking endpoint (Jina/Cohere convention).", "implementation": "Class `RerankResponse` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 516, "end_line": 521, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L516-L521", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ChatCompletionChunkDelta", "qualname": "ChatCompletionChunkDelta", "full_name": "vllm_mlx.api.models.ChatCompletionChunkDelta", "kind": "class", "signature": "class ChatCompletionChunkDelta(BaseModel)", "parameters": [ { "name": "role", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "content", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "reasoning_content", "kind": "field", "annotation": "str | None", "default": "Field(default=None, validation_alias=AliasChoices('reasoning_content', 'reasoning'))", "required": false, "description": "Optional constructor field; defaults to `Field(default=None, validation_alias=AliasChoices('reasoning_content', 'reasoning'))`." }, { "name": "tool_calls", "kind": "field", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ChatCompletionChunkDelta", "docstring": "Delta content in a streaming chunk.", "summary": "Delta content in a streaming chunk.", "implementation": "Class `ChatCompletionChunkDelta` derives from `BaseModel` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 529, "end_line": 562, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L529-L562", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reasoning", "qualname": "ChatCompletionChunkDelta.reasoning", "full_name": "vllm_mlx.api.models.ChatCompletionChunkDelta.reasoning", "kind": "method", "signature": "def reasoning(self) -> str | None", "parameters": [], "return_annotation": "str | None", "docstring": "Return incremental reasoning through the compatibility alias.", "summary": "Return incremental reasoning through the compatibility alias.", "implementation": "Method `ChatCompletionChunkDelta.reasoning` returns `self.reasoning_content`.", "documented": true, "public": true, "addressable": true, "line": 541, "end_line": 544, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L541-L544", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.reasoning_content" ], "state_writes": [], "raises": [], "return_expressions": [ "self.reasoning_content" ], "awaits": false, "yields": false }, { "name": "_serialize", "qualname": "ChatCompletionChunkDelta._serialize", "full_name": "vllm_mlx.api.models.ChatCompletionChunkDelta._serialize", "kind": "method", "signature": "def _serialize(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Serialize delta with only non-None fields.\n\nPer OpenAI streaming spec, delta objects only include fields that\ncarry new content.", "summary": "Serialize delta with only non-None fields.", "implementation": "Method `ChatCompletionChunkDelta._serialize` returns `d`.", "documented": true, "public": false, "addressable": true, "line": 547, "end_line": 562, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L547-L562", "decorators": [ "model_serializer" ], "calls": [], "state_reads": [ "self.role", "self.content", "self.reasoning_content", "self.tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "d" ], "awaits": false, "yields": false }, { "name": "ChatCompletionChunkChoice", "qualname": "ChatCompletionChunkChoice", "full_name": "vllm_mlx.api.models.ChatCompletionChunkChoice", "kind": "class", "signature": "class ChatCompletionChunkChoice(BaseModel)", "parameters": [ { "name": "index", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "delta", "kind": "field", "annotation": "ChatCompletionChunkDelta", "default": "", "required": true, "description": "Required constructor field." }, { "name": "finish_reason", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ChatCompletionChunkChoice", "docstring": "A single choice in a streaming chunk.", "summary": "A single choice in a streaming chunk.", "implementation": "Class `ChatCompletionChunkChoice` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 565, "end_line": 570, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L565-L570", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ChatCompletionChunk", "qualname": "ChatCompletionChunk", "full_name": "vllm_mlx.api.models.ChatCompletionChunk", "kind": "class", "signature": "class ChatCompletionChunk(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str", "default": "Field(default_factory=lambda: f'chatcmpl-{uuid.uuid4().hex[:8]}')", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: f'chatcmpl-{uuid.uuid4().hex[:8]}')`." }, { "name": "object", "kind": "field", "annotation": "str", "default": "'chat.completion.chunk'", "required": false, "description": "Optional constructor field; defaults to `'chat.completion.chunk'`." }, { "name": "created", "kind": "field", "annotation": "int", "default": "Field(default_factory=lambda: int(time.time()))", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: int(time.time()))`." }, { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "choices", "kind": "field", "annotation": "list[ChatCompletionChunkChoice]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "usage", "kind": "field", "annotation": "Usage | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ChatCompletionChunk", "docstring": "A streaming chunk for chat completion.", "summary": "A streaming chunk for chat completion.", "implementation": "Class `ChatCompletionChunk` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 573, "end_line": 581, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/models.py#L573-L581", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.prompt_canonicalize", "path": "vllm_mlx/api/prompt_canonicalize.py", "page_path": "reference/api/vllm_mlx/api/prompt_canonicalize.md", "docstring": "System-prompt canonicalization helpers.", "summary": "System-prompt canonicalization helpers.", "line_count": 51, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/prompt_canonicalize.py#L1-L51", "members": [ "_STRIPPERS", "canonicalize_system_prompt", "canonicalize_system_messages" ], "symbols": [ { "name": "canonicalize_system_prompt", "qualname": "canonicalize_system_prompt", "full_name": "vllm_mlx.api.prompt_canonicalize.canonicalize_system_prompt", "kind": "function", "signature": "def canonicalize_system_prompt(text: str | None) -> str | None", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "Remove known non-semantic volatile lines from system prompt text.", "summary": "Remove known non-semantic volatile lines from system prompt text.", "implementation": "Function `canonicalize_system_prompt` calls `pattern.sub`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 17, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/prompt_canonicalize.py#L17-L24", "decorators": [], "calls": [ "pattern.sub" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "text" ], "awaits": false, "yields": false }, { "name": "canonicalize_system_messages", "qualname": "canonicalize_system_messages", "full_name": "vllm_mlx.api.prompt_canonicalize.canonicalize_system_messages", "kind": "function", "signature": "def canonicalize_system_messages(messages: list[dict]) -> list[dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Canonicalize string content on system-role messages without mutation.", "summary": "Canonicalize string content on system-role messages without mutation.", "implementation": "Function `canonicalize_system_messages` calls `message.get`, `canonicalized.append`, `isinstance`, `canonicalize_system_prompt`; returns `canonicalized if changed else messages`.", "documented": true, "public": true, "addressable": true, "line": 27, "end_line": 51, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/prompt_canonicalize.py#L27-L51", "decorators": [], "calls": [ "message.get", "canonicalized.append", "isinstance", "canonicalize_system_prompt", "message.copy" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "canonicalized if changed else messages" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.responses_models", "path": "vllm_mlx/api/responses_models.py", "page_path": "reference/api/vllm_mlx/api/responses_models.md", "docstring": "Pydantic models for the OpenAI-compatible Responses API.\n\nThis intentionally implements the subset needed for local coding-agent\nworkflows: text messages, function tools, function call outputs, and SSE\nstreaming events. The object and event shapes follow the conventions used by\nOpenAI's gpt-oss reference server and llama.cpp's OpenAI-compatible server.", "summary": "Pydantic models for the OpenAI-compatible Responses API.", "line_count": 342, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L1-L342", "members": [ "ResponseTextFormat", "ResponseTextConfig", "ResponseReasoningConfig", "ResponseTextContentPart", "ResponseReasoningTextPart", "ResponseReasoningSummaryTextPart", "ResponseMessageItem", "ResponseReasoningItem", "ResponseFunctionCallItem", "ResponseFunctionCallOutputItem", "ResponseFunctionTool", "ResponsesInputTokenDetails", "ResponsesOutputTokenDetails", "ResponsesUsage", "ResponseError", "ResponseIncompleteDetails", "ResponsesRequest", "ResponseObject", "ResponsesEventBase", "ResponseCreatedEvent", "ResponseInProgressEvent", "ResponseCompletedEvent", "ResponseOutputItemAddedEvent", "ResponseOutputItemDoneEvent", "ResponseContentPartAddedEvent", "ResponseContentPartDoneEvent", "ResponseOutputTextDeltaEvent", "ResponseOutputTextDoneEvent", "ResponseReasoningTextDeltaEvent", "ResponseReasoningTextDoneEvent", "ResponseFunctionCallArgumentsDeltaEvent" ], "symbols": [ { "name": "ResponseTextFormat", "qualname": "ResponseTextFormat", "full_name": "vllm_mlx.api.responses_models.ResponseTextFormat", "kind": "class", "signature": "class ResponseTextFormat(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "Literal['text', 'json_object']", "default": "'text'", "required": false, "description": "Optional constructor field; defaults to `'text'`." } ], "return_annotation": "ResponseTextFormat", "docstring": "Output text format configuration.", "summary": "Output text format configuration.", "implementation": "Class `ResponseTextFormat` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 19, "end_line": 22, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L19-L22", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseTextConfig", "qualname": "ResponseTextConfig", "full_name": "vllm_mlx.api.responses_models.ResponseTextConfig", "kind": "class", "signature": "class ResponseTextConfig(BaseModel)", "parameters": [ { "name": "format", "kind": "field", "annotation": "ResponseTextFormat", "default": "Field(default_factory=ResponseTextFormat)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=ResponseTextFormat)`." } ], "return_annotation": "ResponseTextConfig", "docstring": "Text output configuration.", "summary": "Text output configuration.", "implementation": "Class `ResponseTextConfig` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 25, "end_line": 28, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L25-L28", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseReasoningConfig", "qualname": "ResponseReasoningConfig", "full_name": "vllm_mlx.api.responses_models.ResponseReasoningConfig", "kind": "class", "signature": "class ResponseReasoningConfig(BaseModel)", "parameters": [ { "name": "effort", "kind": "field", "annotation": "Literal['none', 'minimal', 'low', 'medium', 'high', 'xhigh'] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ResponseReasoningConfig", "docstring": "Reasoning configuration.", "summary": "Reasoning configuration.", "implementation": "Class `ResponseReasoningConfig` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 31, "end_line": 34, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L31-L34", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseTextContentPart", "qualname": "ResponseTextContentPart", "full_name": "vllm_mlx.api.responses_models.ResponseTextContentPart", "kind": "class", "signature": "class ResponseTextContentPart(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "Literal['text', 'input_text', 'output_text']", "default": "'output_text'", "required": false, "description": "Optional constructor field; defaults to `'output_text'`." }, { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "annotations", "kind": "field", "annotation": "list[dict]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." }, { "name": "logprobs", "kind": "field", "annotation": "list[dict]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." } ], "return_annotation": "ResponseTextContentPart", "docstring": "A text content part for message items.", "summary": "A text content part for message items.", "implementation": "Class `ResponseTextContentPart` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 37, "end_line": 43, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L37-L43", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseReasoningTextPart", "qualname": "ResponseReasoningTextPart", "full_name": "vllm_mlx.api.responses_models.ResponseReasoningTextPart", "kind": "class", "signature": "class ResponseReasoningTextPart(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "Literal['reasoning_text']", "default": "'reasoning_text'", "required": false, "description": "Optional constructor field; defaults to `'reasoning_text'`." }, { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ResponseReasoningTextPart", "docstring": "A reasoning text content part.", "summary": "A reasoning text content part.", "implementation": "Class `ResponseReasoningTextPart` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 46, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L46-L50", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseReasoningSummaryTextPart", "qualname": "ResponseReasoningSummaryTextPart", "full_name": "vllm_mlx.api.responses_models.ResponseReasoningSummaryTextPart", "kind": "class", "signature": "class ResponseReasoningSummaryTextPart(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "Literal['summary_text']", "default": "'summary_text'", "required": false, "description": "Optional constructor field; defaults to `'summary_text'`." }, { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ResponseReasoningSummaryTextPart", "docstring": "A reasoning summary item.", "summary": "A reasoning summary item.", "implementation": "Class `ResponseReasoningSummaryTextPart` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 53, "end_line": 57, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L53-L57", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseMessageItem", "qualname": "ResponseMessageItem", "full_name": "vllm_mlx.api.responses_models.ResponseMessageItem", "kind": "class", "signature": "class ResponseMessageItem(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "type", "kind": "field", "annotation": "Literal['message']", "default": "'message'", "required": false, "description": "Optional constructor field; defaults to `'message'`." }, { "name": "role", "kind": "field", "annotation": "Literal['system', 'user', 'assistant', 'developer']", "default": "'assistant'", "required": false, "description": "Optional constructor field; defaults to `'assistant'`." }, { "name": "content", "kind": "field", "annotation": "str | list[ResponseTextContentPart]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." }, { "name": "status", "kind": "field", "annotation": "Literal['in_progress', 'completed', 'incomplete'] | None", "default": "'completed'", "required": false, "description": "Optional constructor field; defaults to `'completed'`." } ], "return_annotation": "ResponseMessageItem", "docstring": "A Responses API message item.", "summary": "A Responses API message item.", "implementation": "Class `ResponseMessageItem` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 60, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L60-L67", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseReasoningItem", "qualname": "ResponseReasoningItem", "full_name": "vllm_mlx.api.responses_models.ResponseReasoningItem", "kind": "class", "signature": "class ResponseReasoningItem(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "type", "kind": "field", "annotation": "Literal['reasoning']", "default": "'reasoning'", "required": false, "description": "Optional constructor field; defaults to `'reasoning'`." }, { "name": "summary", "kind": "field", "annotation": "list[ResponseReasoningSummaryTextPart]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." }, { "name": "content", "kind": "field", "annotation": "list[ResponseReasoningTextPart]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." }, { "name": "status", "kind": "field", "annotation": "Literal['in_progress', 'completed', 'incomplete'] | None", "default": "'completed'", "required": false, "description": "Optional constructor field; defaults to `'completed'`." } ], "return_annotation": "ResponseReasoningItem", "docstring": "A reasoning output item.", "summary": "A reasoning output item.", "implementation": "Class `ResponseReasoningItem` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 77, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L70-L77", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseFunctionCallItem", "qualname": "ResponseFunctionCallItem", "full_name": "vllm_mlx.api.responses_models.ResponseFunctionCallItem", "kind": "class", "signature": "class ResponseFunctionCallItem(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "type", "kind": "field", "annotation": "Literal['function_call']", "default": "'function_call'", "required": false, "description": "Optional constructor field; defaults to `'function_call'`." }, { "name": "call_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "arguments", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "status", "kind": "field", "annotation": "Literal['in_progress', 'completed', 'incomplete']", "default": "'completed'", "required": false, "description": "Optional constructor field; defaults to `'completed'`." } ], "return_annotation": "ResponseFunctionCallItem", "docstring": "A function call output item.", "summary": "A function call output item.", "implementation": "Class `ResponseFunctionCallItem` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 80, "end_line": 88, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L80-L88", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseFunctionCallOutputItem", "qualname": "ResponseFunctionCallOutputItem", "full_name": "vllm_mlx.api.responses_models.ResponseFunctionCallOutputItem", "kind": "class", "signature": "class ResponseFunctionCallOutputItem(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "Literal['function_call_output']", "default": "'function_call_output'", "required": false, "description": "Optional constructor field; defaults to `'function_call_output'`." }, { "name": "call_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "output", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ResponseFunctionCallOutputItem", "docstring": "A tool result item passed back into a later request.", "summary": "A tool result item passed back into a later request.", "implementation": "Class `ResponseFunctionCallOutputItem` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 91, "end_line": 96, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L91-L96", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseFunctionTool", "qualname": "ResponseFunctionTool", "full_name": "vllm_mlx.api.responses_models.ResponseFunctionTool", "kind": "class", "signature": "class ResponseFunctionTool(BaseModel)", "parameters": [ { "name": "type", "kind": "field", "annotation": "Literal['function']", "default": "'function'", "required": false, "description": "Optional constructor field; defaults to `'function'`." }, { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str | None", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "parameters", "kind": "field", "annotation": "dict", "default": "Field(default_factory=lambda: {'type': 'object', 'properties': {}})", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: {'type': 'object', 'properties': {}})`." }, { "name": "strict", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "ResponseFunctionTool", "docstring": "A function tool definition.", "summary": "A function tool definition.", "implementation": "Class `ResponseFunctionTool` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 99, "end_line": 108, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L99-L108", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponsesInputTokenDetails", "qualname": "ResponsesInputTokenDetails", "full_name": "vllm_mlx.api.responses_models.ResponsesInputTokenDetails", "kind": "class", "signature": "class ResponsesInputTokenDetails(BaseModel)", "parameters": [ { "name": "cached_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "ResponsesInputTokenDetails", "docstring": "Input token breakdown.", "summary": "Input token breakdown.", "implementation": "Class `ResponsesInputTokenDetails` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 111, "end_line": 114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L111-L114", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponsesOutputTokenDetails", "qualname": "ResponsesOutputTokenDetails", "full_name": "vllm_mlx.api.responses_models.ResponsesOutputTokenDetails", "kind": "class", "signature": "class ResponsesOutputTokenDetails(BaseModel)", "parameters": [ { "name": "reasoning_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "ResponsesOutputTokenDetails", "docstring": "Output token breakdown.", "summary": "Output token breakdown.", "implementation": "Class `ResponsesOutputTokenDetails` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 117, "end_line": 120, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L117-L120", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponsesUsage", "qualname": "ResponsesUsage", "full_name": "vllm_mlx.api.responses_models.ResponsesUsage", "kind": "class", "signature": "class ResponsesUsage(BaseModel)", "parameters": [ { "name": "input_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "output_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "input_tokens_details", "kind": "field", "annotation": "ResponsesInputTokenDetails", "default": "Field(default_factory=ResponsesInputTokenDetails)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=ResponsesInputTokenDetails)`." }, { "name": "output_tokens_details", "kind": "field", "annotation": "ResponsesOutputTokenDetails", "default": "Field(default_factory=ResponsesOutputTokenDetails)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=ResponsesOutputTokenDetails)`." } ], "return_annotation": "ResponsesUsage", "docstring": "Responses API token usage.", "summary": "Responses API token usage.", "implementation": "Class `ResponsesUsage` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 123, "end_line": 134, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L123-L134", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseError", "qualname": "ResponseError", "full_name": "vllm_mlx.api.responses_models.ResponseError", "kind": "class", "signature": "class ResponseError(BaseModel)", "parameters": [ { "name": "code", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "message", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ResponseError", "docstring": "Error payload.", "summary": "Error payload.", "implementation": "Class `ResponseError` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 137, "end_line": 141, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L137-L141", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseIncompleteDetails", "qualname": "ResponseIncompleteDetails", "full_name": "vllm_mlx.api.responses_models.ResponseIncompleteDetails", "kind": "class", "signature": "class ResponseIncompleteDetails(BaseModel)", "parameters": [ { "name": "reason", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ResponseIncompleteDetails", "docstring": "Incomplete response details.", "summary": "Incomplete response details.", "implementation": "Class `ResponseIncompleteDetails` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 144, "end_line": 147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L144-L147", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponsesRequest", "qualname": "ResponsesRequest", "full_name": "vllm_mlx.api.responses_models.ResponsesRequest", "kind": "class", "signature": "class ResponsesRequest(BaseModel)", "parameters": [ { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "input", "kind": "field", "annotation": "str | list[ResponseMessageItem | ResponseReasoningItem | ResponseFunctionCallItem | ResponseFunctionCallOutputItem | di…", "default": "", "required": true, "description": "Required constructor field." }, { "name": "instructions", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_output_tokens", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "stream", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "tools", "kind": "field", "annotation": "list[ResponseFunctionTool | dict]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." }, { "name": "tool_choice", "kind": "field", "annotation": "str | dict | None", "default": "'auto'", "required": false, "description": "Optional constructor field; defaults to `'auto'`." }, { "name": "parallel_tool_calls", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "previous_response_id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "temperature", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "top_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "chat_template_kwargs", "kind": "field", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "metadata", "kind": "field", "annotation": "dict", "default": "Field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=dict)`." }, { "name": "text", "kind": "field", "annotation": "ResponseTextConfig", "default": "Field(default_factory=ResponseTextConfig)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=ResponseTextConfig)`." }, { "name": "reasoning", "kind": "field", "annotation": "ResponseReasoningConfig | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "store", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "truncation", "kind": "field", "annotation": "str", "default": "'disabled'", "required": false, "description": "Optional constructor field; defaults to `'disabled'`." }, { "name": "user", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ResponsesRequest", "docstring": "Request payload for /v1/responses.", "summary": "Request payload for /v1/responses.", "implementation": "Class `ResponsesRequest` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 150, "end_line": 179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L150-L179", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseObject", "qualname": "ResponseObject", "full_name": "vllm_mlx.api.responses_models.ResponseObject", "kind": "class", "signature": "class ResponseObject(BaseModel)", "parameters": [ { "name": "id", "kind": "field", "annotation": "str", "default": "Field(default_factory=lambda: f'resp_{uuid.uuid4().hex}')", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: f'resp_{uuid.uuid4().hex}')`." }, { "name": "object", "kind": "field", "annotation": "Literal['response']", "default": "'response'", "required": false, "description": "Optional constructor field; defaults to `'response'`." }, { "name": "created_at", "kind": "field", "annotation": "int", "default": "Field(default_factory=lambda: int(time.time()))", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=lambda: int(time.time()))`." }, { "name": "status", "kind": "field", "annotation": "Literal['completed', 'failed', 'incomplete', 'in_progress']", "default": "'completed'", "required": false, "description": "Optional constructor field; defaults to `'completed'`." }, { "name": "background", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "error", "kind": "field", "annotation": "ResponseError | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "incomplete_details", "kind": "field", "annotation": "ResponseIncompleteDetails | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "instructions", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_output_tokens", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_tool_calls", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "metadata", "kind": "field", "annotation": "dict", "default": "Field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=dict)`." }, { "name": "model", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "output", "kind": "field", "annotation": "list[ResponseMessageItem | ResponseReasoningItem | ResponseFunctionCallItem]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." }, { "name": "parallel_tool_calls", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "previous_response_id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "text", "kind": "field", "annotation": "ResponseTextConfig", "default": "Field(default_factory=ResponseTextConfig)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=ResponseTextConfig)`." }, { "name": "tool_choice", "kind": "field", "annotation": "str | dict | None", "default": "'auto'", "required": false, "description": "Optional constructor field; defaults to `'auto'`." }, { "name": "tools", "kind": "field", "annotation": "list[ResponseFunctionTool | dict]", "default": "Field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `Field(default_factory=list)`." }, { "name": "top_p", "kind": "field", "annotation": "float", "default": "1.0", "required": false, "description": "Optional constructor field; defaults to `1.0`." }, { "name": "temperature", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "truncation", "kind": "field", "annotation": "str", "default": "'disabled'", "required": false, "description": "Optional constructor field; defaults to `'disabled'`." }, { "name": "usage", "kind": "field", "annotation": "ResponsesUsage | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "user", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "store", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." } ], "return_annotation": "ResponseObject", "docstring": "Response object for /v1/responses.", "summary": "Response object for /v1/responses.", "implementation": "Class `ResponseObject` derives from `BaseModel` and declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 182, "end_line": 226, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L182-L226", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "output_text", "qualname": "ResponseObject.output_text", "full_name": "vllm_mlx.api.responses_models.ResponseObject.output_text", "kind": "method", "signature": "def output_text(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Concatenate assistant text content into the convenience field.", "summary": "Concatenate assistant text content into the convenience field.", "implementation": "Method `ResponseObject.output_text` calls `isinstance`, `text_parts.append`, `''.join`; returns `''.join(text_parts)`.", "documented": true, "public": true, "addressable": true, "line": 214, "end_line": 226, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L214-L226", "decorators": [ "computed_field", "property" ], "calls": [ "isinstance", "text_parts.append", "''.join" ], "state_reads": [ "self.output" ], "state_writes": [], "raises": [], "return_expressions": [ "''.join(text_parts)" ], "awaits": false, "yields": false }, { "name": "ResponsesEventBase", "qualname": "ResponsesEventBase", "full_name": "vllm_mlx.api.responses_models.ResponsesEventBase", "kind": "class", "signature": "class ResponsesEventBase(BaseModel)", "parameters": [ { "name": "sequence_number", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ResponsesEventBase", "docstring": "Base event fields.", "summary": "Base event fields.", "implementation": "Class `ResponsesEventBase` derives from `BaseModel` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 229, "end_line": 232, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L229-L232", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseCreatedEvent", "qualname": "ResponseCreatedEvent", "full_name": "vllm_mlx.api.responses_models.ResponseCreatedEvent", "kind": "class", "signature": "class ResponseCreatedEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseCreatedEvent", "docstring": "Signal that a response object has been created.", "summary": "Signal that a response object has been created.", "implementation": "Class `ResponseCreatedEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 235, "end_line": 239, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L235-L239", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseInProgressEvent", "qualname": "ResponseInProgressEvent", "full_name": "vllm_mlx.api.responses_models.ResponseInProgressEvent", "kind": "class", "signature": "class ResponseInProgressEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseInProgressEvent", "docstring": "Signal that response generation is in progress.", "summary": "Signal that response generation is in progress.", "implementation": "Class `ResponseInProgressEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 242, "end_line": 246, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L242-L246", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseCompletedEvent", "qualname": "ResponseCompletedEvent", "full_name": "vllm_mlx.api.responses_models.ResponseCompletedEvent", "kind": "class", "signature": "class ResponseCompletedEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseCompletedEvent", "docstring": "Carry the terminal completed response object.", "summary": "Carry the terminal completed response object.", "implementation": "Class `ResponseCompletedEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 249, "end_line": 253, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L249-L253", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseOutputItemAddedEvent", "qualname": "ResponseOutputItemAddedEvent", "full_name": "vllm_mlx.api.responses_models.ResponseOutputItemAddedEvent", "kind": "class", "signature": "class ResponseOutputItemAddedEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseOutputItemAddedEvent", "docstring": "Announce a newly added response output item.", "summary": "Announce a newly added response output item.", "implementation": "Class `ResponseOutputItemAddedEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 256, "end_line": 261, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L256-L261", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseOutputItemDoneEvent", "qualname": "ResponseOutputItemDoneEvent", "full_name": "vllm_mlx.api.responses_models.ResponseOutputItemDoneEvent", "kind": "class", "signature": "class ResponseOutputItemDoneEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseOutputItemDoneEvent", "docstring": "Signal that a response output item is complete.", "summary": "Signal that a response output item is complete.", "implementation": "Class `ResponseOutputItemDoneEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 264, "end_line": 269, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L264-L269", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseContentPartAddedEvent", "qualname": "ResponseContentPartAddedEvent", "full_name": "vllm_mlx.api.responses_models.ResponseContentPartAddedEvent", "kind": "class", "signature": "class ResponseContentPartAddedEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseContentPartAddedEvent", "docstring": "Announce a content part attached to an output item.", "summary": "Announce a content part attached to an output item.", "implementation": "Class `ResponseContentPartAddedEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 272, "end_line": 279, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L272-L279", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseContentPartDoneEvent", "qualname": "ResponseContentPartDoneEvent", "full_name": "vllm_mlx.api.responses_models.ResponseContentPartDoneEvent", "kind": "class", "signature": "class ResponseContentPartDoneEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseContentPartDoneEvent", "docstring": "Signal that an output item's content part is complete.", "summary": "Signal that an output item's content part is complete.", "implementation": "Class `ResponseContentPartDoneEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 282, "end_line": 289, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L282-L289", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseOutputTextDeltaEvent", "qualname": "ResponseOutputTextDeltaEvent", "full_name": "vllm_mlx.api.responses_models.ResponseOutputTextDeltaEvent", "kind": "class", "signature": "class ResponseOutputTextDeltaEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseOutputTextDeltaEvent", "docstring": "Carry an incremental final-answer text fragment.", "summary": "Carry an incremental final-answer text fragment.", "implementation": "Class `ResponseOutputTextDeltaEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 292, "end_line": 300, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L292-L300", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseOutputTextDoneEvent", "qualname": "ResponseOutputTextDoneEvent", "full_name": "vllm_mlx.api.responses_models.ResponseOutputTextDoneEvent", "kind": "class", "signature": "class ResponseOutputTextDoneEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseOutputTextDoneEvent", "docstring": "Carry the complete final-answer text for one content part.", "summary": "Carry the complete final-answer text for one content part.", "implementation": "Class `ResponseOutputTextDoneEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 303, "end_line": 311, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L303-L311", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseReasoningTextDeltaEvent", "qualname": "ResponseReasoningTextDeltaEvent", "full_name": "vllm_mlx.api.responses_models.ResponseReasoningTextDeltaEvent", "kind": "class", "signature": "class ResponseReasoningTextDeltaEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseReasoningTextDeltaEvent", "docstring": "Carry an incremental reasoning text fragment.", "summary": "Carry an incremental reasoning text fragment.", "implementation": "Class `ResponseReasoningTextDeltaEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 314, "end_line": 321, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L314-L321", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseReasoningTextDoneEvent", "qualname": "ResponseReasoningTextDoneEvent", "full_name": "vllm_mlx.api.responses_models.ResponseReasoningTextDoneEvent", "kind": "class", "signature": "class ResponseReasoningTextDoneEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseReasoningTextDoneEvent", "docstring": "Carry the complete reasoning text for one content part.", "summary": "Carry the complete reasoning text for one content part.", "implementation": "Class `ResponseReasoningTextDoneEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 324, "end_line": 331, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L324-L331", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResponseFunctionCallArgumentsDeltaEvent", "qualname": "ResponseFunctionCallArgumentsDeltaEvent", "full_name": "vllm_mlx.api.responses_models.ResponseFunctionCallArgumentsDeltaEvent", "kind": "class", "signature": "class ResponseFunctionCallArgumentsDeltaEvent(ResponsesEventBase)", "parameters": [], "return_annotation": "ResponseFunctionCallArgumentsDeltaEvent", "docstring": "Carry an incremental fragment of function-call arguments.", "summary": "Carry an incremental fragment of function-call arguments.", "implementation": "Class `ResponseFunctionCallArgumentsDeltaEvent` derives from `ResponsesEventBase` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 334, "end_line": 342, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/responses_models.py#L334-L342", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.streaming", "path": "vllm_mlx/api/streaming.py", "page_path": "reference/api/vllm_mlx/api/streaming.md", "docstring": "Optimized streaming JSON encoder for SSE responses.\n\nThis module provides a pre-computed template-based JSON encoder that reduces\nCPU overhead during streaming by avoiding repeated json.dumps() calls for\nstatic parts of the response.\n\nPerformance improvement: ~20-30% reduction in server CPU overhead for streaming.", "summary": "Optimized streaming JSON encoder for SSE responses.", "line_count": 210, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/streaming.py#L1-L210", "members": [ "_escape_json_string", "StreamingJSONEncoder" ], "symbols": [ { "name": "_escape_json_string", "qualname": "_escape_json_string", "full_name": "vllm_mlx.api.streaming._escape_json_string", "kind": "function", "signature": "def _escape_json_string(s: str) -> str", "parameters": [ { "name": "s", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Escape a string for JSON without the surrounding quotes.\n\nUses json.dumps for correctness then strips the quotes.\nThis handles all special characters: quotes, backslashes, newlines, tabs, unicode.", "summary": "Escape a string for JSON without the surrounding quotes.", "implementation": "Function `_escape_json_string` calls `json.dumps`; returns `json.dumps(s)[1:-1]`.", "documented": true, "public": false, "addressable": true, "line": 16, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/streaming.py#L16-L24", "decorators": [], "calls": [ "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.dumps(s)[1:-1]" ], "awaits": false, "yields": false }, { "name": "StreamingJSONEncoder", "qualname": "StreamingJSONEncoder", "full_name": "vllm_mlx.api.streaming.StreamingJSONEncoder", "kind": "class", "signature": "class StreamingJSONEncoder", "parameters": [ { "name": "response_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Unique response ID (e.g., \"chatcmpl-abc123\")" }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Model name (e.g., \"mlx-community/Llama-3.2-3B-Instruct-4bit\")" }, { "name": "object_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Response object type (\"text_completion\" or \"chat.completion.chunk\")" }, { "name": "created", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Unix timestamp (defaults to current time)" } ], "return_annotation": "StreamingJSONEncoder", "docstring": "Optimized JSON encoder for OpenAI-compatible streaming responses.\n\nPre-computes static parts of the JSON response at initialization time,\nthen only inserts dynamic content (text/content, finish_reason) per token.\n\nThe main optimization is pre-building the static JSON prefix and suffix\nthat don't change between tokens. Only the dynamic parts (content, finish_reason)\nare escaped and inserted per token.\n\nExample usage:\n encoder = StreamingJSONEncoder(\n response_id=\"chatcmpl-123\",\n model=\"gpt-4\",\n object_type=\"chat.completion.chunk\"\n )\n\n # Encode each token\n for token in tokens:\n yield encoder.encode_chat_chunk(content=token)\n\n # Final chunk with finish_reason\n yield encoder.encode_chat_chunk(finish_reason=\"stop\")\n yield encoder.encode_done()", "summary": "Optimized JSON encoder for OpenAI-compatible streaming responses.", "implementation": "Class `StreamingJSONEncoder` declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 27, "end_line": 210, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/streaming.py#L27-L210", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "StreamingJSONEncoder.__init__", "full_name": "vllm_mlx.api.streaming.StreamingJSONEncoder.__init__", "kind": "method", "signature": "def __init__(self, response_id: str, model: str, object_type: str, created: int | None=None)", "parameters": [ { "name": "response_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Unique response ID (e.g., \"chatcmpl-abc123\")" }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Model name (e.g., \"mlx-community/Llama-3.2-3B-Instruct-4bit\")" }, { "name": "object_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Response object type (\"text_completion\" or \"chat.completion.chunk\")" }, { "name": "created", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Unix timestamp (defaults to current time)" } ], "return_annotation": "not annotated", "docstring": "Initialize the encoder with static response metadata.\n\nPre-computes template parts that don't change between tokens.\n\nArgs:\n response_id: Unique response ID (e.g., \"chatcmpl-abc123\")\n model: Model name (e.g., \"mlx-community/Llama-3.2-3B-Instruct-4bit\")\n object_type: Response object type (\"text_completion\" or \"chat.completion.chunk\")\n created: Unix timestamp (defaults to current time)", "summary": "Initialize the encoder with static response metadata.", "implementation": "Method `StreamingJSONEncoder.__init__` updates `self.response_id`, `self.model`, `self.object_type`, `self.created`; calls `int`, `time.time`, `_escape_json_string`.", "documented": true, "public": false, "addressable": true, "line": 57, "end_line": 102, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/streaming.py#L57-L102", "decorators": [], "calls": [ "int", "time.time", "_escape_json_string" ], "state_reads": [ "self.created" ], "state_writes": [ "self.response_id", "self.model", "self.object_type", "self.created", "self._prefix", "self._completion_choices_prefix", "self._completion_text_prefix", "self._completion_text_suffix", "self._chat_choices_prefix", "self._chat_finish_prefix" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "encode_completion_chunk", "qualname": "StreamingJSONEncoder.encode_completion_chunk", "full_name": "vllm_mlx.api.streaming.StreamingJSONEncoder.encode_completion_chunk", "kind": "method", "signature": "def encode_completion_chunk(self, text: str, index: int=0, finish_reason: str | None=None, usage: dict[str, int] | None=None) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The generated text for this chunk" }, { "name": "index", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Choice index (usually 0)" }, { "name": "finish_reason", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "\"stop\", \"length\", or None if not finished" }, { "name": "usage", "kind": "positional or keyword", "annotation": "dict[str, int] | None", "default": "None", "required": false, "description": "Optional usage stats (prompt_tokens, completion_tokens, total_tokens)" } ], "return_annotation": "str", "docstring": " Encode a text completion chunk using pre-computed templates.\n\n Args:\n text: The generated text for this chunk\n index: Choice index (usually 0)\n finish_reason: \"stop\", \"length\", or None if not finished\n usage: Optional usage stats (prompt_tokens, completion_tokens, total_tokens)\n\n Returns:\n SSE-formatted string: \"data: {json}\n\n\"\n ", "summary": "Encode a text completion chunk using pre-computed templates.", "implementation": "Method `StreamingJSONEncoder.encode_completion_chunk` calls `_escape_json_string`, `json.dumps`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 104, "end_line": 149, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/streaming.py#L104-L149", "decorators": [], "calls": [ "_escape_json_string", "json.dumps" ], "state_reads": [ "self._completion_choices_prefix", "self._completion_text_prefix", "self._completion_text_suffix", "self._prefix" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "encode_chat_chunk", "qualname": "StreamingJSONEncoder.encode_chat_chunk", "full_name": "vllm_mlx.api.streaming.StreamingJSONEncoder.encode_chat_chunk", "kind": "method", "signature": "def encode_chat_chunk(self, role: str | None=None, content: str | None=None, finish_reason: str | None=None, usage: dict[str, int] | None=None) -> str", "parameters": [ { "name": "role", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Assistant role (only for first chunk)" }, { "name": "content", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Generated content for this chunk" }, { "name": "finish_reason", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "\"stop\", \"length\", or None if not finished" }, { "name": "usage", "kind": "positional or keyword", "annotation": "dict[str, int] | None", "default": "None", "required": false, "description": "Optional usage stats" } ], "return_annotation": "str", "docstring": " Encode a chat completion chunk using pre-computed templates.\n\n Args:\n role: Assistant role (only for first chunk)\n content: Generated content for this chunk\n finish_reason: \"stop\", \"length\", or None if not finished\n usage: Optional usage stats\n\n Returns:\n SSE-formatted string: \"data: {json}\n\n\"\n ", "summary": "Encode a chat completion chunk using pre-computed templates.", "implementation": "Method `StreamingJSONEncoder.encode_chat_chunk` calls `delta_parts.append`, `_escape_json_string`, `','.join`, `json.dumps`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 151, "end_line": 201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/streaming.py#L151-L201", "decorators": [], "calls": [ "delta_parts.append", "_escape_json_string", "','.join", "json.dumps" ], "state_reads": [ "self._chat_choices_prefix", "self._chat_finish_prefix", "self._prefix" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "encode_done", "qualname": "StreamingJSONEncoder.encode_done", "full_name": "vllm_mlx.api.streaming.StreamingJSONEncoder.encode_done", "kind": "method", "signature": "def encode_done(self) -> str", "parameters": [], "return_annotation": "str", "docstring": " Encode the [DONE] message that signals end of stream.\n\n Returns:\n SSE-formatted done message: \"data: [DONE]\n\n\"\n ", "summary": "Encode the [DONE] message that signals end of stream.", "implementation": "Method `StreamingJSONEncoder.encode_done` returns `self._DONE_MSG`.", "documented": true, "public": true, "addressable": true, "line": 203, "end_line": 210, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/streaming.py#L203-L210", "decorators": [], "calls": [], "state_reads": [ "self._DONE_MSG" ], "state_writes": [], "raises": [], "return_expressions": [ "self._DONE_MSG" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.tool_calling", "path": "vllm_mlx/api/tool_calling.py", "page_path": "reference/api/vllm_mlx/api/tool_calling.md", "docstring": "Tool calling parsing and conversion utilities.\n\nSupports parsing tool calls from multiple model formats:\n- Qwen: {\"name\": \"...\", \"arguments\": {...}}\n- Llama: {\"arg\": \"value\"}\n\nAlso includes structured output (JSON Schema) utilities:\n- parse_json_output: Extract JSON from model output\n- validate_json_schema: Validate JSON against a schema", "summary": "Tool calling parsing and conversion utilities.", "line_count": 1035, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L1-L1035", "members": [ "InvalidResponseFormatOutput", "_looks_like_tool_call", "_parse_raw_json_tool_calls", "parse_tool_calls", "convert_tools_for_template", "format_tool_call_for_message", "validate_json_schema", "_scan_balanced_json", "_repair_truncated_json", "extract_json_from_text", "StreamingJsonFenceStripper", "parse_json_output", "apply_response_format_or_error", "build_json_system_prompt", "build_json_logits_processor" ], "symbols": [ { "name": "InvalidResponseFormatOutput", "qualname": "InvalidResponseFormatOutput", "full_name": "vllm_mlx.api.tool_calling.InvalidResponseFormatOutput", "kind": "class", "signature": "class InvalidResponseFormatOutput(ValueError)", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "InvalidResponseFormatOutput", "docstring": "Raised when generated content does not satisfy response_format.", "summary": "Raised when generated content does not satisfy response_format.", "implementation": "Class `InvalidResponseFormatOutput` derives from `ValueError` and declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 24, "end_line": 29, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L24-L29", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "InvalidResponseFormatOutput.__init__", "full_name": "vllm_mlx.api.tool_calling.InvalidResponseFormatOutput.__init__", "kind": "method", "signature": "def __init__(self, message: str)", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `InvalidResponseFormatOutput.__init__` updates `self.message`; calls `super().__init__`, `super`.", "implementation": "Method `InvalidResponseFormatOutput.__init__` updates `self.message`; calls `super().__init__`, `super`.", "documented": false, "public": false, "addressable": true, "line": 27, "end_line": 29, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L27-L29", "decorators": [], "calls": [ "super().__init__", "super" ], "state_reads": [], "state_writes": [ "self.message" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_looks_like_tool_call", "qualname": "_looks_like_tool_call", "full_name": "vllm_mlx.api.tool_calling._looks_like_tool_call", "kind": "function", "signature": "def _looks_like_tool_call(obj: Any) -> bool", "parameters": [ { "name": "obj", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Parsed JSON object." } ], "return_annotation": "bool", "docstring": "Heuristic: decide whether a parsed JSON object really represents a tool\ncall as opposed to user data that happens to carry a ``\"name\"`` field.\n\nThe OpenAI tool-call wire format ALWAYS has both ``\"name\"`` and\n``\"arguments\"``. Accepting bare ``{\"name\": ...}`` (previous behaviour)\ncaused ``response_format={\"type\": \"json_schema\"}`` payloads with a\n``name`` field to be hijacked as fake tool calls (observed on\nMiniMax-M2: ``{\"name\": \"John\", \"age\": 25}`` -> ``function.name=\"John\"``).\n\nArgs:\n obj: Parsed JSON object.\n\nReturns:\n True if obj looks like a tool call, False otherwise.", "summary": "Heuristic: decide whether a parsed JSON object really represents a tool call as opposed to user data that happens to carry a ``\"name\"`` field.", "implementation": "Function `_looks_like_tool_call` calls `isinstance`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 32, "end_line": 57, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L32-L57", "decorators": [], "calls": [ "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "isinstance(args, (dict, str))" ], "awaits": false, "yields": false }, { "name": "_parse_raw_json_tool_calls", "qualname": "_parse_raw_json_tool_calls", "full_name": "vllm_mlx.api.tool_calling._parse_raw_json_tool_calls", "kind": "function", "signature": "def _parse_raw_json_tool_calls(text: str) -> Optional[List[dict]]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Raw model output text" } ], "return_annotation": "Optional[List[dict]]", "docstring": "Parse raw JSON tool calls from model output.\n\nHandles:\n- Single JSON object: {\"name\": \"func\", \"arguments\": {...}}\n- Multiple objects separated by commas: {...}, {...}\n- JSON array: [{...}, {...}]\n\nOnly accepts objects that carry both ``name`` AND ``arguments`` fields\nto avoid hijacking user data emitted via ``response_format``.\n\nArgs:\n text: Raw model output text\n\nReturns:\n List of tool call dicts with 'name' and 'arguments', or None if no valid tool calls found", "summary": "Parse raw JSON tool calls from model output.", "implementation": "Function `_parse_raw_json_tool_calls` calls `text.strip`, `text.startswith`, `json.loads`, `isinstance`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 60, "end_line": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L60-L123", "decorators": [], "calls": [ "text.strip", "text.startswith", "json.loads", "isinstance", "all", "_looks_like_tool_call", "enumerate", "tool_calls.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "[{'name': item['name'], 'arguments': item['arguments']} for item in parsed]", "tool_calls if tool_calls else None" ], "awaits": false, "yields": false }, { "name": "parse_tool_calls", "qualname": "parse_tool_calls", "full_name": "vllm_mlx.api.tool_calling.parse_tool_calls", "kind": "function", "signature": "def parse_tool_calls(text: str, request: dict[str, Any] | None=None) -> Tuple[str, Optional[List[ToolCall]]]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Raw model output text" }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "Tuple[str, Optional[List[ToolCall]]]", "docstring": "Parse tool calls from model output.\n\nSupports multiple formats:\n- MiniMax: v\n- Qwen3 bracket: [Calling tool: function_name({\"arg\": \"value\"})]\n- Qwen: {\"name\": \"...\", \"arguments\": {...}}\n- Llama: {\"arg\": \"value\"}\n- Nemotron: v\n- Raw JSON: {\"name\": \"...\", \"arguments\": {...}} (single or multiple)\n\nArgs:\n text: Raw model output text\n\nReturns:\n Tuple of (cleaned_text, tool_calls or None)\n - cleaned_text: Text with tool call tags removed\n - tool_calls: List of ToolCall objects, or None if no tool calls found", "summary": "Parse tool calls from model output.", "implementation": "Function `parse_tool_calls` calls `re.findall`, `json.loads`, `tool_calls.append`, `ToolCall`; returns `(cleaned_text, tool_calls if tool_calls else None)`.", "documented": true, "public": true, "addressable": true, "line": 126, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L126-L351", "decorators": [], "calls": [ "re.findall", "json.loads", "tool_calls.append", "ToolCall", "uuid.uuid4", "FunctionCall", "name.strip", "json.dumps", "re.sub('\\\\s*.*?\\\\s*', '', cleaned_text, flags=re.DOTALL).strip", "re.sub", "isinstance", "str", "re.sub('\\\\[Calling tool:\\\\s*\\\\w+\\\\(\\\\{.*?\\\\}\\\\)\\\\]', '', cleaned_text, flags=re.DOTALL).strip", "p_value.strip", "p_name.strip", "re.sub('\\\\s*]+>.*?\\\\s*', '', text, flags=re.DOTALL).strip", "data.get", "re.sub('\\\\s*\\\\{.*?\\\\}\\\\s*', '', cleaned_text, flags=re.DOTALL).strip", "re.sub(']+>\\\\{.*?\\\\}', '', cleaned_text, flags=re.DOTALL).strip", "bool", "request.get", "_parse_raw_json_tool_calls" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(cleaned_text, tool_calls if tool_calls else None)" ], "awaits": false, "yields": false }, { "name": "convert_tools_for_template", "qualname": "convert_tools_for_template", "full_name": "vllm_mlx.api.tool_calling.convert_tools_for_template", "kind": "function", "signature": "def convert_tools_for_template(tools: Optional[List]) -> Optional[List[dict]]", "parameters": [ { "name": "tools", "kind": "positional or keyword", "annotation": "Optional[List]", "default": "", "required": true, "description": "List of ToolDefinition objects or dicts in OpenAI format" } ], "return_annotation": "Optional[List[dict]]", "docstring": "Convert OpenAI tools format to format expected by tokenizer.apply_chat_template.\n\nOpenAI format:\n[{\"type\": \"function\", \"function\": {\"name\": \"...\", \"description\": \"...\", \"parameters\": {...}}}]\n\nTemplate format (commonly used by models):\n[{\"type\": \"function\", \"function\": {\"name\": \"...\", \"description\": \"...\", \"parameters\": {...}}}]\n\nArgs:\n tools: List of ToolDefinition objects or dicts in OpenAI format\n\nReturns:\n List of tool definitions in template format, or None if no tools", "summary": "Convert OpenAI tools format to format expected by tokenizer.apply_chat_template.", "implementation": "Function `convert_tools_for_template` calls `isinstance`, `tool.get`, `getattr`, `tool_func.get`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 354, "end_line": 409, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L354-L409", "decorators": [], "calls": [ "isinstance", "tool.get", "getattr", "tool_func.get", "converted.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "converted if converted else None" ], "awaits": false, "yields": false }, { "name": "format_tool_call_for_message", "qualname": "format_tool_call_for_message", "full_name": "vllm_mlx.api.tool_calling.format_tool_call_for_message", "kind": "function", "signature": "def format_tool_call_for_message(tool_call: ToolCall) -> dict", "parameters": [ { "name": "tool_call", "kind": "positional or keyword", "annotation": "ToolCall", "default": "", "required": true, "description": "ToolCall object" } ], "return_annotation": "dict", "docstring": "Format a ToolCall object for inclusion in a message.\n\nArgs:\n tool_call: ToolCall object\n\nReturns:\n Dict representation suitable for message content", "summary": "Format a ToolCall object for inclusion in a message.", "implementation": "Function `format_tool_call_for_message` returns `{'id': tool_call.id, 'type': tool_call.type, 'function': {'name': tool_call.function.name, 'arguments': tool_call.funct…`.", "documented": true, "public": true, "addressable": true, "line": 412, "end_line": 429, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L412-L429", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'id': tool_call.id, 'type': tool_call.type, 'function': {'name': tool_call.function.name, 'arguments': tool_call.funct…" ], "awaits": false, "yields": false }, { "name": "validate_json_schema", "qualname": "validate_json_schema", "full_name": "vllm_mlx.api.tool_calling.validate_json_schema", "kind": "function", "signature": "def validate_json_schema(data: Any, schema: Dict[str, Any]) -> Tuple[bool, Optional[str]]", "parameters": [ { "name": "data", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The JSON data to validate (dict, list, etc.)" }, { "name": "schema", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "JSON Schema specification" } ], "return_annotation": "Tuple[bool, Optional[str]]", "docstring": "Validate JSON data against a JSON Schema.\n\nArgs:\n data: The JSON data to validate (dict, list, etc.)\n schema: JSON Schema specification\n\nReturns:\n Tuple of (is_valid, error_message)\n - is_valid: True if data matches schema\n - error_message: Error description if invalid, None if valid", "summary": "Validate JSON data against a JSON Schema.", "implementation": "Function `validate_json_schema` calls `validate`, `str`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 437, "end_line": 456, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L437-L456", "decorators": [], "calls": [ "validate", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(True, None)", "(False, str(e.message))" ], "awaits": false, "yields": false }, { "name": "_scan_balanced_json", "qualname": "_scan_balanced_json", "full_name": "vllm_mlx.api.tool_calling._scan_balanced_json", "kind": "function", "signature": "def _scan_balanced_json(text: str, start: int) -> Optional[str]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "start", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[str]", "docstring": "Walk forward from ``start`` (which must point at ``{`` or ``[``) and\nreturn the substring that represents the first balanced JSON value,\nrespecting strings and escapes. Returns ``None`` if the opening bracket\nis never closed (truncated output).", "summary": "Walk forward from ``start`` (which must point at ``{`` or ``[``) and return the substring that represents the first balanced JSON value, respecting strings and escapes.", "implementation": "Function `_scan_balanced_json` calls `len`, `range`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 459, "end_line": 494, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L459-L494", "decorators": [], "calls": [ "len", "range" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "text[start:i + 1]" ], "awaits": false, "yields": false }, { "name": "_repair_truncated_json", "qualname": "_repair_truncated_json", "full_name": "vllm_mlx.api.tool_calling._repair_truncated_json", "kind": "function", "signature": "def _repair_truncated_json(fragment: str) -> Optional[Dict[str, Any]]", "parameters": [ { "name": "fragment", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[Dict[str, Any]]", "docstring": "Attempt to parse a JSON fragment whose closing brackets were cut off\n(e.g. because the model hit ``max_tokens`` mid-object).\n\nStrategy: scan once to determine the open-bracket stack and whether we\nended mid-string, then try a handful of repair candidates in order of\nlikelihood:\n\n 1. Close unterminated string, close brackets.\n 2. Also strip a dangling ``,`` / ``:`` before closing.\n 3. Also drop a dangling key (``\"k\":`` or bare ``\"k\"``) before closing.\n 4. Drop a dangling partial token (number / true / fals / nul) before\n closing.\n\nReturns the first candidate that ``json.loads`` accepts, or ``None``.", "summary": "Attempt to parse a JSON fragment whose closing brackets were cut off (e.g.", "implementation": "Function `_repair_truncated_json` calls `stack.append`, `stack.pop`, `candidates.append`, `_close`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 497, "end_line": 582, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L497-L582", "decorators": [], "calls": [ "stack.append", "stack.pop", "candidates.append", "_close", "re.sub", "json.loads" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "json.loads(candidate)" ], "awaits": false, "yields": false }, { "name": "_close", "qualname": "_repair_truncated_json._close", "full_name": "vllm_mlx.api.tool_calling._repair_truncated_json._close", "kind": "nested function", "signature": "def _close(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Nested Function `_repair_truncated_json._close` calls `reversed`; returns `text`.", "implementation": "Nested Function `_repair_truncated_json._close` calls `reversed`; returns `text`.", "documented": false, "public": false, "addressable": false, "line": 540, "end_line": 543, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L540-L543", "decorators": [], "calls": [ "reversed" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text" ], "awaits": false, "yields": false }, { "name": "extract_json_from_text", "qualname": "extract_json_from_text", "full_name": "vllm_mlx.api.tool_calling.extract_json_from_text", "kind": "function", "signature": "def extract_json_from_text(text: str) -> Optional[Dict[str, Any]]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Raw model output text" } ], "return_annotation": "Optional[Dict[str, Any]]", "docstring": "Extract JSON from model output text.\n\nTries multiple strategies, in order of specificity:\n\n1. Parse entire text as JSON\n2. Extract JSON from complete markdown code blocks (``` ... ```)\n3. Extract JSON from an unterminated markdown code block (``` json\\n{ ... )\n — handles the common \"chatty + truncation\" failure mode where the\n model starts a ```json fence, never closes it, then hits max_tokens.\n4. Balanced-brace scan for the first ``{`` or ``[`` in the text\n5. Repair truncated JSON by closing unclosed brackets/strings\n\nArgs:\n text: Raw model output text\n\nReturns:\n Parsed JSON data, or None if no valid JSON found", "summary": "Extract JSON from model output text.", "implementation": "Function `extract_json_from_text` calls `text.strip`, `json.loads`, `re.findall`, `match.strip`; has 6 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 585, "end_line": 666, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L585-L666", "decorators": [], "calls": [ "text.strip", "json.loads", "re.findall", "match.strip", "re.search", "unterminated_fence.group(1).strip", "unterminated_fence.group", "fenced_candidate.endswith", "fenced_candidate[:-3].strip", "text.find", "_scan_balanced_json", "candidates.append", "_repair_truncated_json" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.loads(text)", "json.loads(match.strip())", "json.loads(fenced_candidate)", "json.loads(candidate)", "repaired", "None" ], "awaits": false, "yields": false }, { "name": "StreamingJsonFenceStripper", "qualname": "StreamingJsonFenceStripper", "full_name": "vllm_mlx.api.tool_calling.StreamingJsonFenceStripper", "kind": "class", "signature": "class StreamingJsonFenceStripper", "parameters": [], "return_annotation": "StreamingJsonFenceStripper", "docstring": "Strip markdown code fences from streamed content when response_format is set.\n\nWithout guided decoding, chat models often wrap their JSON output in markdown\nfences (```json ... ```) even when the system prompt says not to. The non-\nstreaming path strips those via ``extract_json_from_text`` / ``parse_json_output``,\nbut the streaming path used to emit the raw deltas, so clients got\n``\"```json{...}```\"`` instead of ``\"{...}\"``.\n\nThis filter buffers just enough text to detect:\n * a leading fence like ``\"```\"``, ``\"```json\"``, ``\"```\\n\"`` or\n ``\"```json\\n\"`` (with optional leading whitespace), possibly split\n across SSE deltas, and\n * a trailing fence like ``\"```\"`` or ``\"\\n```\\n\"`` on stream end.\n\nLeading-whitespace and leading fences are consumed; trailing fences are\ndropped in :meth:`finalize`. Non-fenced content passes through with at most\na ``_TAIL_HOLDBACK``-char delay.", "summary": "Strip markdown code fences from streamed content when response_format is set.", "implementation": "Class `StreamingJsonFenceStripper` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 669, "end_line": 787, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L669-L787", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "StreamingJsonFenceStripper.__init__", "full_name": "vllm_mlx.api.tool_calling.StreamingJsonFenceStripper.__init__", "kind": "method", "signature": "def __init__(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `StreamingJsonFenceStripper.__init__` updates `self._buf`, `self._past_opening`.", "implementation": "Method `StreamingJsonFenceStripper.__init__` updates `self._buf`, `self._past_opening`.", "documented": false, "public": false, "addressable": true, "line": 695, "end_line": 697, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L695-L697", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._buf", "self._past_opening" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "feed", "qualname": "StreamingJsonFenceStripper.feed", "full_name": "vllm_mlx.api.tool_calling.StreamingJsonFenceStripper.feed", "kind": "method", "signature": "def feed(self, delta: str) -> str", "parameters": [ { "name": "delta", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Append a content delta and return the portion safe to emit now.", "summary": "Append a content delta and return the portion safe to emit now.", "implementation": "Method `StreamingJsonFenceStripper.feed` updates `self._buf`, `self._past_opening`; calls `self._buf.lstrip`, `len`, `opening.startswith`, `ls.startswith`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 699, "end_line": 748, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L699-L748", "decorators": [], "calls": [ "self._buf.lstrip", "len", "opening.startswith", "ls.startswith", "ls[len(matched):].lstrip", "buf[i - 1].isspace", "min" ], "state_reads": [ "self._past_opening", "self._buf.lstrip", "self._buf", "self._OPENINGS", "self._TAIL_HOLDBACK" ], "state_writes": [ "self._buf", "self._past_opening" ], "raises": [], "return_expressions": [ "''", "to_emit" ], "awaits": false, "yields": false }, { "name": "finalize", "qualname": "StreamingJsonFenceStripper.finalize", "full_name": "vllm_mlx.api.tool_calling.StreamingJsonFenceStripper.finalize", "kind": "method", "signature": "def finalize(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Flush the remaining buffer, dropping any trailing fence.", "summary": "Flush the remaining buffer, dropping any trailing fence.", "implementation": "Method `StreamingJsonFenceStripper.finalize` updates `self._buf`, `self._past_opening`; calls `tail.lstrip`, `len`, `opening.startswith`, `ls.startswith`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 750, "end_line": 787, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L750-L787", "decorators": [], "calls": [ "tail.lstrip", "len", "opening.startswith", "ls.startswith", "ls[len(matched):].lstrip", "tail.rstrip", "stripped.endswith", "stripped[:-len(closing)].rstrip" ], "state_reads": [ "self._buf", "self._past_opening", "self._OPENINGS" ], "state_writes": [ "self._buf", "self._past_opening" ], "raises": [], "return_expressions": [ "''", "stripped[:-len(closing)].rstrip()", "tail" ], "awaits": false, "yields": false }, { "name": "parse_json_output", "qualname": "parse_json_output", "full_name": "vllm_mlx.api.tool_calling.parse_json_output", "kind": "function", "signature": "def parse_json_output(text: str, response_format: Optional[Union[ResponseFormat, Dict[str, Any]]]=None) -> Tuple[str, Optional[Dict[str, Any]], bool, Optional[str]]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Raw model output text" }, { "name": "response_format", "kind": "positional or keyword", "annotation": "Optional[Union[ResponseFormat, Dict[str, Any]]]", "default": "None", "required": false, "description": "ResponseFormat specification (optional) - If type=\"json_object\", extracts any valid JSON - If type=\"json_schema\", extracts and validates against schema" } ], "return_annotation": "Tuple[str, Optional[Dict[str, Any]], bool, Optional[str]]", "docstring": "Parse JSON from model output when response_format is set.\n\nArgs:\n text: Raw model output text\n response_format: ResponseFormat specification (optional)\n - If type=\"json_object\", extracts any valid JSON\n - If type=\"json_schema\", extracts and validates against schema\n\nReturns:\n Tuple of (cleaned_text, parsed_json, is_valid, error_message)\n - cleaned_text: Original text (preserved for reference)\n - parsed_json: Extracted JSON data, or None if extraction failed\n - is_valid: True if JSON is valid (and matches schema if specified)\n - error_message: Error description if invalid, None if valid", "summary": "Parse JSON from model output when response_format is set.", "implementation": "Function `parse_json_output` calls `isinstance`, `rf_dict.get`, `extract_json_from_text`, `json_schema_spec.get`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 790, "end_line": 855, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L790-L855", "decorators": [], "calls": [ "isinstance", "rf_dict.get", "extract_json_from_text", "json_schema_spec.get", "validate_json_schema" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(text, None, True, None)", "(text, None, False, 'Failed to extract valid JSON from output')", "(text, parsed, True, None)", "(text, parsed, False, f'JSON Schema validation failed: {error}')" ], "awaits": false, "yields": false }, { "name": "apply_response_format_or_error", "qualname": "apply_response_format_or_error", "full_name": "vllm_mlx.api.tool_calling.apply_response_format_or_error", "kind": "function", "signature": "def apply_response_format_or_error(text: str, response_format: object, *, ensure_ascii: bool=False) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "response_format", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "ensure_ascii", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." } ], "return_annotation": "str", "docstring": "Return canonical JSON content or raise for invalid response_format output.", "summary": "Return canonical JSON content or raise for invalid response_format output.", "implementation": "Function `apply_response_format_or_error` calls `parse_json_output`, `json.dumps`, `InvalidResponseFormatOutput`; can raise `InvalidResponseFormatOutput`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 858, "end_line": 873, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L858-L873", "decorators": [], "calls": [ "parse_json_output", "json.dumps", "InvalidResponseFormatOutput" ], "state_reads": [], "state_writes": [], "raises": [ "InvalidResponseFormatOutput" ], "return_expressions": [ "json.dumps(parsed_json, ensure_ascii=ensure_ascii)", "text" ], "awaits": false, "yields": false }, { "name": "build_json_system_prompt", "qualname": "build_json_system_prompt", "full_name": "vllm_mlx.api.tool_calling.build_json_system_prompt", "kind": "function", "signature": "def build_json_system_prompt(response_format: Optional[Union[ResponseFormat, Dict[str, Any]]]=None, *, thinking_model: bool=False) -> Optional[str]", "parameters": [ { "name": "response_format", "kind": "positional or keyword", "annotation": "Optional[Union[ResponseFormat, Dict[str, Any]]]", "default": "None", "required": false, "description": "ResponseFormat specification" }, { "name": "thinking_model", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "When ``True`` use softer output rules that allow the model to reason (think) before emitting JSON. Strict rules that demand ``{`` as the very first character conflict with ```` blocks and cause degenerated output." } ], "return_annotation": "Optional[str]", "docstring": "Build a system prompt instruction for JSON output.\n\nFor models without native JSON mode support, this adds instructions\nto the prompt to encourage proper JSON formatting.\n\nArgs:\n response_format: ResponseFormat specification\n thinking_model: When ``True`` use softer output rules that allow\n the model to reason (think) before emitting JSON. Strict rules\n that demand ``{`` as the very first character conflict with\n ```` blocks and cause degenerated output.\n\nReturns:\n System prompt instruction string, or None if not needed", "summary": "Build a system prompt instruction for JSON output.", "implementation": "Function `build_json_system_prompt` calls `isinstance`, `rf_dict.get`, `json_schema_spec.get`, `json.dumps`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 876, "end_line": 953, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L876-L953", "decorators": [], "calls": [ "isinstance", "rf_dict.get", "json_schema_spec.get", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "'You must respond with a single valid JSON value only.\\n\\n' + strict_rules", "prompt" ], "awaits": false, "yields": false }, { "name": "build_json_logits_processor", "qualname": "build_json_logits_processor", "full_name": "vllm_mlx.api.tool_calling.build_json_logits_processor", "kind": "function", "signature": "def build_json_logits_processor(response_format: ResponseFormat | dict[str, Any] | None, tokenizer: Any)", "parameters": [ { "name": "response_format", "kind": "positional or keyword", "annotation": "ResponseFormat | dict[str, Any] | None", "default": "", "required": true, "description": "``ResponseFormat`` specification (or dict)." }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The tokenizer used by the engine. May be a HF tokenizer, a ``mlx_lm.TokenizerWrapper``, or a VLM ``processor``; the underlying tokenizer is resolved automatically." } ], "return_annotation": "not annotated", "docstring": "Build a logits processor that constrains generation to valid JSON matching\n``response_format``.\n\nUnlike :func:`build_json_system_prompt` which nudges the model via the\nsystem prompt, this processor masks logits at every generation step so\nthe model *cannot* emit invalid JSON (grammar-guided decoding).\n\nArgs:\n response_format: ``ResponseFormat`` specification (or dict).\n tokenizer: The tokenizer used by the engine. May be a HF tokenizer,\n a ``mlx_lm.TokenizerWrapper``, or a VLM ``processor``; the\n underlying tokenizer is resolved automatically.\n\nReturns:\n A callable ``(tokens, logits) -> logits`` suitable for passing to\n ``mlx_lm.stream_generate`` via ``logits_processors``. ``None`` when\n no constraint is needed (e.g. ``type=text``) or when constrained\n decoding cannot be enabled (missing optional dependency, tokenizer\n incompatibility) — in that case the caller should fall back to the\n system-prompt path.", "summary": "Build a logits processor that constrains generation to valid JSON matching ``response_format``.", "implementation": "Function `build_json_logits_processor` calls `isinstance`, `response_format.get`, `json_schema_spec.get`, `getattr`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 956, "end_line": 1035, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/tool_calling.py#L956-L1035", "decorators": [], "calls": [ "isinstance", "response_format.get", "json_schema_spec.get", "getattr", "is_available", "JSONSchemaLogitsProcessor" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "JSONSchemaLogitsProcessor(schema=schema, tokenizer=tokenizer)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.api.utils", "path": "vllm_mlx/api/utils.py", "page_path": "reference/api/vllm_mlx/api/utils.md", "docstring": "Utility functions for text processing and model detection.", "summary": "Utility functions for text processing and model detection.", "line_count": 747, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L1-L747", "members": [ "logger", "SPECIAL_TOKENS_PATTERN", "_FINAL_CHANNEL_RE", "_clean_gpt_oss_output", "clean_output_text", "_MAX_TOOL_BUFFER_BYTES", "_TOOL_CALL_TAGS", "StreamingToolCallFilter", "StreamingThinkRouter", "MLLM_PATTERNS", "_VLM_CONFIG_KEYS", "_VLM_ARCHITECTURE_KEYWORDS", "_MAX_CONFIG_JSON_BYTES", "_try_read_config_json", "_config_indicates_vlm", "_check_legacy_string_patterns", "is_mllm_model", "is_vlm_model", "MEDIA_CONTENT_TYPES", "has_media_content", "_content_to_text", "extract_multimodal_content" ], "symbols": [ { "name": "_clean_gpt_oss_output", "qualname": "_clean_gpt_oss_output", "full_name": "vllm_mlx.api.utils._clean_gpt_oss_output", "kind": "function", "signature": "def _clean_gpt_oss_output(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Raw model output containing channel tokens." } ], "return_annotation": "str", "docstring": "Extract final channel content from GPT-OSS channel-based output.\n\nWhen reasoning parser is not enabled, this provides a fallback that\nextracts the 'final' channel content so the API response is usable.\n\nHandles both standard and extended format with constrain token:\n <|channel|>final<|message|>...\n <|channel|>final <|constrain|>JSON<|message|>...\n\nArgs:\n text: Raw model output containing channel tokens.\n\nReturns:\n Extracted final content, or text with channel tokens stripped.", "summary": "Extract final channel content from GPT-OSS channel-based output.", "implementation": "Function `_clean_gpt_oss_output` calls `_FINAL_CHANNEL_RE.search`, `match.end`, `re.sub`, `content.strip`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 39, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L39-L73", "decorators": [], "calls": [ "_FINAL_CHANNEL_RE.search", "match.end", "re.sub", "content.strip", "cleaned.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "content.strip()", "cleaned.strip()" ], "awaits": false, "yields": false }, { "name": "clean_output_text", "qualname": "clean_output_text", "full_name": "vllm_mlx.api.utils.clean_output_text", "kind": "function", "signature": "def clean_output_text(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Raw model output" } ], "return_annotation": "str", "docstring": "Clean model output by removing special tokens.\n\nKeeps ... blocks intact for reasoning models.\nAdds opening tag if missing (happens when thinking is enabled\nin the prompt template but the tag is part of the prompt, not output).\nHandles GPT-OSS channel-based format as fallback when reasoning parser\nis not enabled.\n\nArgs:\n text: Raw model output\n\nReturns:\n Cleaned text with special tokens removed", "summary": "Clean model output by removing special tokens.", "implementation": "Function `clean_output_text` calls `_clean_gpt_oss_output`, `SPECIAL_TOKENS_PATTERN.sub`, `text.strip`, `text.lstrip().startswith`; returns `text`.", "documented": true, "public": true, "addressable": true, "line": 76, "end_line": 108, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L76-L108", "decorators": [], "calls": [ "_clean_gpt_oss_output", "SPECIAL_TOKENS_PATTERN.sub", "text.strip", "text.lstrip().startswith", "text.lstrip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text" ], "awaits": false, "yields": false }, { "name": "StreamingToolCallFilter", "qualname": "StreamingToolCallFilter", "full_name": "vllm_mlx.api.utils.StreamingToolCallFilter", "kind": "class", "signature": "class StreamingToolCallFilter", "parameters": [], "return_annotation": "StreamingToolCallFilter", "docstring": "Buffer streaming text to suppress tool call markup.\n\nTool call XML (e.g. ...) arrives\nsplit across multiple streaming deltas. This filter detects entry into a\ntool call block, suppresses all output until the block closes, and emits\nonly non-tool-call text.\n\nThe full unfiltered text is still accumulated separately for tool call\nparsing at stream end.", "summary": "Buffer streaming text to suppress tool call markup.", "implementation": "Class `StreamingToolCallFilter` declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 134, "end_line": 229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L134-L229", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "StreamingToolCallFilter.__init__", "full_name": "vllm_mlx.api.utils.StreamingToolCallFilter.__init__", "kind": "method", "signature": "def __init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `StreamingToolCallFilter.__init__` updates `self._buffer`, `self._in_block`, `self._close_tag`, `self._max_open_len`; calls `max`, `len`.", "implementation": "Method `StreamingToolCallFilter.__init__` updates `self._buffer`, `self._in_block`, `self._close_tag`, `self._max_open_len`; calls `max`, `len`.", "documented": false, "public": false, "addressable": true, "line": 146, "end_line": 151, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L146-L151", "decorators": [], "calls": [ "max", "len" ], "state_reads": [], "state_writes": [ "self._buffer", "self._in_block", "self._close_tag", "self._max_open_len" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "process", "qualname": "StreamingToolCallFilter.process", "full_name": "vllm_mlx.api.utils.StreamingToolCallFilter.process", "kind": "method", "signature": "def process(self, delta: str) -> str", "parameters": [ { "name": "delta", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Process a streaming delta. Returns text to emit (may be empty).", "summary": "Process a streaming delta.", "implementation": "Method `StreamingToolCallFilter.process` updates `self._buffer`; calls `self._consume_block`, `self._scan_for_open`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 153, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L153-L160", "decorators": [], "calls": [ "self._consume_block", "self._scan_for_open" ], "state_reads": [ "self._in_block", "self._consume_block", "self._scan_for_open" ], "state_writes": [ "self._buffer" ], "raises": [], "return_expressions": [ "self._consume_block()", "self._scan_for_open()" ], "awaits": false, "yields": false }, { "name": "_scan_for_open", "qualname": "StreamingToolCallFilter._scan_for_open", "full_name": "vllm_mlx.api.utils.StreamingToolCallFilter._scan_for_open", "kind": "method", "signature": "def _scan_for_open(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Scan buffer for tool call open tags. Emit safe text.", "summary": "Scan buffer for tool call open tags.", "implementation": "Method `StreamingToolCallFilter._scan_for_open` updates `self._buffer`, `self._in_block`, `self._close_tag`; calls `self._buffer.find`, `len`, `self._consume_block`, `range`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 162, "end_line": 194, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L162-L194", "decorators": [], "calls": [ "self._buffer.find", "len", "self._consume_block", "range", "min", "self._buffer.endswith", "max" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._consume_block", "self._buffer.endswith" ], "state_writes": [ "self._buffer", "self._in_block", "self._close_tag" ], "raises": [], "return_expressions": [ "emit + after", "emit" ], "awaits": false, "yields": false }, { "name": "_consume_block", "qualname": "StreamingToolCallFilter._consume_block", "full_name": "vllm_mlx.api.utils.StreamingToolCallFilter._consume_block", "kind": "method", "signature": "def _consume_block(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Consume content inside a tool call block. Returns empty string\nunless the block closes and there's text after it.", "summary": "Consume content inside a tool call block.", "implementation": "Method `StreamingToolCallFilter._consume_block` updates `self._buffer`, `self._in_block`, `self._close_tag`; calls `self._buffer.find`, `len`, `self._scan_for_open`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 196, "end_line": 218, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L196-L218", "decorators": [], "calls": [ "self._buffer.find", "len", "self._scan_for_open", "logger.warning" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._close_tag", "self._scan_for_open" ], "state_writes": [ "self._buffer", "self._in_block", "self._close_tag" ], "raises": [], "return_expressions": [ "self._scan_for_open()", "''" ], "awaits": false, "yields": false }, { "name": "flush", "qualname": "StreamingToolCallFilter.flush", "full_name": "vllm_mlx.api.utils.StreamingToolCallFilter.flush", "kind": "method", "signature": "def flush(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Flush remaining buffer at end of stream.", "summary": "Flush remaining buffer at end of stream.", "implementation": "Method `StreamingToolCallFilter.flush` updates `self._buffer`, `self._in_block`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 220, "end_line": 229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L220-L229", "decorators": [], "calls": [], "state_reads": [ "self._in_block", "self._buffer" ], "state_writes": [ "self._buffer", "self._in_block" ], "raises": [], "return_expressions": [ "''", "emit" ], "awaits": false, "yields": false }, { "name": "StreamingThinkRouter", "qualname": "StreamingThinkRouter", "full_name": "vllm_mlx.api.utils.StreamingThinkRouter", "kind": "class", "signature": "class StreamingThinkRouter", "parameters": [ { "name": "start_in_thinking", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "If True, assume the model starts in thinking mode (e.g. MiniMax adds to the generation prompt, so the tag never appears in the output stream)." } ], "return_annotation": "StreamingThinkRouter", "docstring": "Route ... content to separate Anthropic thinking blocks.\n\nInstead of emitting thinking content as plain text (where it's\nindistinguishable from the response), this router yields tagged\npieces that the streaming handler can emit as proper Anthropic\ncontent block types.\n\nEach call to process() returns a list of (block_type, text) tuples:\n- (\"thinking\", text) for content inside ...\n- (\"text\", text) for content outside think blocks\n\nArgs:\n start_in_thinking: If True, assume the model starts in thinking\n mode (e.g. MiniMax adds to the generation prompt,\n so the tag never appears in the output stream).", "summary": "Route ... content to separate Anthropic thinking blocks.", "implementation": "Class `StreamingThinkRouter` declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 237, "end_line": 327, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L237-L327", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "StreamingThinkRouter.__init__", "full_name": "vllm_mlx.api.utils.StreamingThinkRouter.__init__", "kind": "method", "signature": "def __init__(self, start_in_thinking: bool=False)", "parameters": [ { "name": "start_in_thinking", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `StreamingThinkRouter.__init__` updates `self._buffer`, `self._in_think`.", "implementation": "Method `StreamingThinkRouter.__init__` updates `self._buffer`, `self._in_think`.", "documented": false, "public": false, "addressable": true, "line": 255, "end_line": 257, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L255-L257", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._buffer", "self._in_think" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "process", "qualname": "StreamingThinkRouter.process", "full_name": "vllm_mlx.api.utils.StreamingThinkRouter.process", "kind": "method", "signature": "def process(self, delta: str) -> list[tuple[str, str]]", "parameters": [ { "name": "delta", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[tuple[str, str]]", "docstring": "Process a delta. Returns list of (block_type, text) pieces.", "summary": "Process a delta.", "implementation": "Method `StreamingThinkRouter.process` updates `self._buffer`; calls `self._extract_pieces`; returns `pieces`.", "documented": true, "public": true, "addressable": true, "line": 259, "end_line": 264, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L259-L264", "decorators": [], "calls": [ "self._extract_pieces" ], "state_reads": [ "self._extract_pieces" ], "state_writes": [ "self._buffer" ], "raises": [], "return_expressions": [ "pieces" ], "awaits": false, "yields": false }, { "name": "_extract_pieces", "qualname": "StreamingThinkRouter._extract_pieces", "full_name": "vllm_mlx.api.utils.StreamingThinkRouter._extract_pieces", "kind": "method", "signature": "def _extract_pieces(self, pieces: list[tuple[str, str]]) -> None", "parameters": [ { "name": "pieces", "kind": "positional or keyword", "annotation": "list[tuple[str, str]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Extract all complete pieces from the buffer.", "summary": "Extract all complete pieces from the buffer.", "implementation": "Method `StreamingThinkRouter._extract_pieces` updates `self._buffer`, `self._in_think`; calls `self._buffer.find`, `len`, `pieces.append`, `range`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 266, "end_line": 317, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L266-L317", "decorators": [], "calls": [ "self._buffer.find", "len", "pieces.append", "range", "min", "self._buffer.endswith" ], "state_reads": [ "self._in_think", "self._buffer.find", "self._buffer", "self._buffer.endswith" ], "state_writes": [ "self._buffer", "self._in_think" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "flush", "qualname": "StreamingThinkRouter.flush", "full_name": "vllm_mlx.api.utils.StreamingThinkRouter.flush", "kind": "method", "signature": "def flush(self) -> list[tuple[str, str]]", "parameters": [], "return_annotation": "list[tuple[str, str]]", "docstring": "Flush remaining buffer at end of stream.", "summary": "Flush remaining buffer at end of stream.", "implementation": "Method `StreamingThinkRouter.flush` updates `self._buffer`, `self._in_think`; calls `pieces.append`; returns `pieces`.", "documented": true, "public": true, "addressable": true, "line": 319, "end_line": 327, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L319-L327", "decorators": [], "calls": [ "pieces.append" ], "state_reads": [ "self._buffer", "self._in_think" ], "state_writes": [ "self._buffer", "self._in_think" ], "raises": [], "return_expressions": [ "pieces" ], "awaits": false, "yields": false }, { "name": "_try_read_config_json", "qualname": "_try_read_config_json", "full_name": "vllm_mlx.api.utils._try_read_config_json", "kind": "function", "signature": "def _try_read_config_json(name_or_path: str) -> dict | None", "parameters": [ { "name": "name_or_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict | None", "docstring": "Read config.json from a local model directory.\n\nReturns None when the input is not a local directory, the directory has\nno config.json, the file is too large, or it cannot be parsed.", "summary": "Read config.json from a local model directory.", "implementation": "Function `_try_read_config_json` calls `Path`, `candidate.is_dir`, `config_path.is_file`, `config_path.stat`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 408, "end_line": 434, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L408-L434", "decorators": [], "calls": [ "Path", "candidate.is_dir", "config_path.is_file", "config_path.stat", "config_path.open", "json.load", "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "data if isinstance(data, dict) else None" ], "awaits": false, "yields": false }, { "name": "_config_indicates_vlm", "qualname": "_config_indicates_vlm", "full_name": "vllm_mlx.api.utils._config_indicates_vlm", "kind": "function", "signature": "def _config_indicates_vlm(config: dict) -> bool", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Inspect a parsed config.json dict for multimodal markers.", "summary": "Inspect a parsed config.json dict for multimodal markers.", "implementation": "Function `_config_indicates_vlm` calls `config.get`, `isinstance`, `arch.lower`, `keyword.lower`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 437, "end_line": 453, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L437-L453", "decorators": [], "calls": [ "config.get", "isinstance", "arch.lower", "keyword.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "_check_legacy_string_patterns", "qualname": "_check_legacy_string_patterns", "full_name": "vllm_mlx.api.utils._check_legacy_string_patterns", "kind": "function", "signature": "def _check_legacy_string_patterns(model_name: str) -> bool", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Validation 1: substring match of MLLM_PATTERNS against the input string.\n\nKept for HF repo IDs (where no local config.json is reachable) and as\na fallback when config.json cannot be read.", "summary": "Validation 1: substring match of MLLM_PATTERNS against the input string.", "implementation": "Function `_check_legacy_string_patterns` calls `model_name.lower`, `pattern.lower`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 456, "end_line": 466, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L456-L466", "decorators": [], "calls": [ "model_name.lower", "pattern.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "is_mllm_model", "qualname": "is_mllm_model", "full_name": "vllm_mlx.api.utils.is_mllm_model", "kind": "function", "signature": "def is_mllm_model(model_name: str) -> bool", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace repo ID or local filesystem path." } ], "return_annotation": "bool", "docstring": "Check if a model name or path indicates a multimodal language model.\n\nTwo complementary validations are run:\n\n1. config.json inspection: when ``model_name`` resolves to a local\n directory containing a readable config.json, inspect the model's\n own metadata (``architectures`` field, ``vision_config``,\n ``audio_config``, etc.). Authoritative when available because it\n reflects what the model actually is, not how it is named on disk.\n\n2. Legacy substring match against ``MLLM_PATTERNS``: applied when no\n config.json is reachable (e.g., a HuggingFace repo ID before the\n weights are downloaded). Preserves the historical behaviour.\n\nArgs:\n model_name: HuggingFace repo ID or local filesystem path.\n\nReturns:\n True if the model is detected as multimodal (MLLM/VLM).", "summary": "Check if a model name or path indicates a multimodal language model.", "implementation": "Function `is_mllm_model` calls `_try_read_config_json`, `_config_indicates_vlm`, `_check_legacy_string_patterns`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 469, "end_line": 493, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L469-L493", "decorators": [], "calls": [ "_try_read_config_json", "_config_indicates_vlm", "_check_legacy_string_patterns" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_config_indicates_vlm(config)", "_check_legacy_string_patterns(model_name)" ], "awaits": false, "yields": false }, { "name": "has_media_content", "qualname": "has_media_content", "full_name": "vllm_mlx.api.utils.has_media_content", "kind": "function", "signature": "def has_media_content(messages: list) -> bool", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if any message contains media content (images, video, audio).\n\nHandles both plain dicts (``msg.get(\"content\")``) and Pydantic-style\nobjects (``msg.content``) so it works in both engine and server contexts.", "summary": "Check if any message contains media content (images, video, audio).", "implementation": "Function `has_media_content` calls `isinstance`, `msg.get`, `getattr`, `part.get`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 516, "end_line": 536, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L516-L536", "decorators": [], "calls": [ "isinstance", "msg.get", "getattr", "part.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "_content_to_text", "qualname": "_content_to_text", "full_name": "vllm_mlx.api.utils._content_to_text", "kind": "function", "signature": "def _content_to_text(content) -> str", "parameters": [ { "name": "content", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Extract text from content that can be str, list[ContentPart], or None.", "summary": "Extract text from content that can be str, list[ContentPart], or None.", "implementation": "Function `_content_to_text` calls `isinstance`, `hasattr`, `item.model_dump`, `item.dict().items`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 544, "end_line": 560, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L544-L560", "decorators": [], "calls": [ "isinstance", "hasattr", "item.model_dump", "item.dict().items", "item.dict", "item.get", "parts.append", "'\\n'.join", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''", "content", "'\\n'.join(parts)", "str(content)" ], "awaits": false, "yields": false }, { "name": "extract_multimodal_content", "qualname": "extract_multimodal_content", "full_name": "vllm_mlx.api.utils.extract_multimodal_content", "kind": "function", "signature": "def extract_multimodal_content(messages: list[Message], preserve_native_format: bool=False) -> tuple[list[dict], list[str], list[str], list[str]]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[Message]", "default": "", "required": true, "description": "List of Message objects" }, { "name": "preserve_native_format", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "If True, preserve native tool message format (role=\"tool\", tool_calls field) instead of converting to text. Required for models with native tool support in chat templates (e.g., Mistral, Llama 3+, DeepSeek V3)." } ], "return_annotation": "tuple[list[dict], list[str], list[str], list[str]]", "docstring": "Extract text content, images, videos, and audio from OpenAI-format messages.\n\nHandles:\n- Simple text messages\n- Multimodal messages with images/videos/audio\n- Tool call messages (assistant with tool_calls)\n- Tool response messages (role=\"tool\")\n\nArgs:\n messages: List of Message objects\n preserve_native_format: If True, preserve native tool message format\n (role=\"tool\", tool_calls field) instead of converting to text.\n Required for models with native tool support in chat templates\n (e.g., Mistral, Llama 3+, DeepSeek V3).\n\nReturns:\n Tuple of (processed_messages, images, videos, audios)\n - processed_messages: List of {\"role\": str, \"content\": str}\n - images: List of image URLs/paths/base64\n - videos: List of video URLs/paths/base64\n - audios: List of audio URLs/paths/base64", "summary": "Extract text content, images, videos, and audio from OpenAI-format messages.", "implementation": "Function `extract_multimodal_content` calls `isinstance`, `msg.get`, `getattr`, `processed_messages.append`; returns `(processed_messages, images, videos, audios)`.", "documented": true, "public": true, "addressable": true, "line": 563, "end_line": 747, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/api/utils.py#L563-L747", "decorators": [], "calls": [ "isinstance", "msg.get", "getattr", "processed_messages.append", "hasattr", "tc.model_dump", "tc.dict", "tc_copy.get", "func.get", "json.loads", "tool_calls_list.append", "_content_to_text", "tc.get", "tool_calls_text.append", "'\\n'.join", "item.model_dump", "item.dict().items", "item.dict", "item.get", "text_parts.append", "images.append", "img_url.get", "videos.append", "vid_url.get", "audios.append", "audio_url.get", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(processed_messages, images, videos, audios)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.attention", "path": "vllm_mlx/attention.py", "page_path": "reference/api/vllm_mlx/attention.md", "docstring": "MLX Attention Backend for vLLM.\n\nThis module provides an attention backend that uses MLX's native\nattention implementation, optimized for Apple Silicon.", "summary": "MLX Attention Backend for vLLM.", "line_count": 245, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L1-L245", "members": [ "logger", "MLXAttentionMetadata", "MLXAttentionBackend", "MLXAttentionImpl", "create_mlx_attention_backend" ], "symbols": [ { "name": "MLXAttentionMetadata", "qualname": "MLXAttentionMetadata", "full_name": "vllm_mlx.attention.MLXAttentionMetadata", "kind": "class", "signature": "class MLXAttentionMetadata", "parameters": [ { "name": "seq_lens", "kind": "field", "annotation": "list[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "max_seq_len", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "num_prefill_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "num_decode_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "block_tables", "kind": "field", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "slot_mapping", "kind": "field", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MLXAttentionMetadata", "docstring": "Metadata for MLX attention computation.", "summary": "Metadata for MLX attention computation.", "implementation": "Class `MLXAttentionMetadata` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 20, "end_line": 39, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L20-L39", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLXAttentionBackend", "qualname": "MLXAttentionBackend", "full_name": "vllm_mlx.attention.MLXAttentionBackend", "kind": "class", "signature": "class MLXAttentionBackend", "parameters": [], "return_annotation": "MLXAttentionBackend", "docstring": "Attention backend using MLX's native attention.\n\nMLX provides optimized attention implementations that run on\nApple Silicon's GPU via Metal. This backend wraps those\nimplementations for use with vLLM.\n\nNote: mlx-lm handles attention internally, so this backend\nprimarily serves as a compatibility layer.", "summary": "Attention backend using MLX's native attention.", "implementation": "Class `MLXAttentionBackend` declares 9 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 42, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L42-L135", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_name", "qualname": "MLXAttentionBackend.get_name", "full_name": "vllm_mlx.attention.MLXAttentionBackend.get_name", "kind": "method", "signature": "def get_name() -> str", "parameters": [], "return_annotation": "str", "docstring": "Return backend name.", "summary": "Return backend name.", "implementation": "Method `MLXAttentionBackend.get_name` returns `'MLX'`.", "documented": true, "public": true, "addressable": true, "line": 55, "end_line": 57, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L55-L57", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'MLX'" ], "awaits": false, "yields": false }, { "name": "get_impl_cls", "qualname": "MLXAttentionBackend.get_impl_cls", "full_name": "vllm_mlx.attention.MLXAttentionBackend.get_impl_cls", "kind": "method", "signature": "def get_impl_cls() -> type", "parameters": [], "return_annotation": "type", "docstring": "Return the implementation class.", "summary": "Return the implementation class.", "implementation": "Method `MLXAttentionBackend.get_impl_cls` returns `MLXAttentionImpl`.", "documented": true, "public": true, "addressable": true, "line": 60, "end_line": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L60-L62", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "MLXAttentionImpl" ], "awaits": false, "yields": false }, { "name": "get_metadata_cls", "qualname": "MLXAttentionBackend.get_metadata_cls", "full_name": "vllm_mlx.attention.MLXAttentionBackend.get_metadata_cls", "kind": "method", "signature": "def get_metadata_cls() -> type", "parameters": [], "return_annotation": "type", "docstring": "Return the metadata class.", "summary": "Return the metadata class.", "implementation": "Method `MLXAttentionBackend.get_metadata_cls` returns `MLXAttentionMetadata`.", "documented": true, "public": true, "addressable": true, "line": 65, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L65-L67", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "MLXAttentionMetadata" ], "awaits": false, "yields": false }, { "name": "get_kv_cache_shape", "qualname": "MLXAttentionBackend.get_kv_cache_shape", "full_name": "vllm_mlx.attention.MLXAttentionBackend.get_kv_cache_shape", "kind": "method", "signature": "def get_kv_cache_shape(num_blocks: int, block_size: int, num_kv_heads: int, head_size: int) -> tuple[int, ...]", "parameters": [ { "name": "num_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of cache blocks" }, { "name": "block_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Tokens per block" }, { "name": "num_kv_heads", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of KV attention heads" }, { "name": "head_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Size of each attention head" } ], "return_annotation": "tuple[int, ...]", "docstring": "Get the shape of KV cache.\n\nArgs:\n num_blocks: Number of cache blocks\n block_size: Tokens per block\n num_kv_heads: Number of KV attention heads\n head_size: Size of each attention head\n\nReturns:\n Shape tuple for KV cache tensor", "summary": "Get the shape of KV cache.", "implementation": "Method `MLXAttentionBackend.get_kv_cache_shape` returns `(num_blocks, block_size, num_kv_heads, head_size)`.", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L70-L89", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(num_blocks, block_size, num_kv_heads, head_size)" ], "awaits": false, "yields": false }, { "name": "get_supported_head_sizes", "qualname": "MLXAttentionBackend.get_supported_head_sizes", "full_name": "vllm_mlx.attention.MLXAttentionBackend.get_supported_head_sizes", "kind": "method", "signature": "def get_supported_head_sizes() -> list[int]", "parameters": [], "return_annotation": "list[int]", "docstring": "Return supported attention head sizes.", "summary": "Return supported attention head sizes.", "implementation": "Method `MLXAttentionBackend.get_supported_head_sizes` returns `[64, 80, 96, 112, 128, 256]`.", "documented": true, "public": true, "addressable": true, "line": 92, "end_line": 94, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L92-L94", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[64, 80, 96, 112, 128, 256]" ], "awaits": false, "yields": false }, { "name": "validate_configuration", "qualname": "MLXAttentionBackend.validate_configuration", "full_name": "vllm_mlx.attention.MLXAttentionBackend.validate_configuration", "kind": "method", "signature": "def validate_configuration(num_heads: int, head_size: int, num_kv_heads: int, dtype: 'torch.dtype', block_size: int, **kwargs) -> list[str]", "parameters": [ { "name": "num_heads", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "head_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_kv_heads", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "dtype", "kind": "positional or keyword", "annotation": "'torch.dtype'", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "block_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "list[str]", "docstring": "Validate attention configuration.\n\nReturns list of error messages (empty if valid).", "summary": "Validate attention configuration.", "implementation": "Method `MLXAttentionBackend.validate_configuration` calls `MLXAttentionBackend.get_supported_head_sizes`, `errors.append`; returns `errors`.", "documented": true, "public": true, "addressable": true, "line": 97, "end_line": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L97-L118", "decorators": [ "staticmethod" ], "calls": [ "MLXAttentionBackend.get_supported_head_sizes", "errors.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "errors" ], "awaits": false, "yields": false }, { "name": "supports_dtype", "qualname": "MLXAttentionBackend.supports_dtype", "full_name": "vllm_mlx.attention.MLXAttentionBackend.supports_dtype", "kind": "method", "signature": "def supports_dtype(dtype: 'torch.dtype') -> bool", "parameters": [ { "name": "dtype", "kind": "positional or keyword", "annotation": "'torch.dtype'", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if dtype is supported.", "summary": "Check if dtype is supported.", "implementation": "Method `MLXAttentionBackend.supports_dtype` returns `dtype in [torch.float16, torch.bfloat16, torch.float32]`.", "documented": true, "public": true, "addressable": true, "line": 121, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L121-L125", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "dtype in [torch.float16, torch.bfloat16, torch.float32]" ], "awaits": false, "yields": false }, { "name": "supports_block_size", "qualname": "MLXAttentionBackend.supports_block_size", "full_name": "vllm_mlx.attention.MLXAttentionBackend.supports_block_size", "kind": "method", "signature": "def supports_block_size(block_size: int) -> bool", "parameters": [ { "name": "block_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if block size is supported.", "summary": "Check if block size is supported.", "implementation": "Method `MLXAttentionBackend.supports_block_size` returns `block_size in [8, 16, 32]`.", "documented": true, "public": true, "addressable": true, "line": 128, "end_line": 130, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L128-L130", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "block_size in [8, 16, 32]" ], "awaits": false, "yields": false }, { "name": "supports_attn_type", "qualname": "MLXAttentionBackend.supports_attn_type", "full_name": "vllm_mlx.attention.MLXAttentionBackend.supports_attn_type", "kind": "method", "signature": "def supports_attn_type(attn_type: str) -> bool", "parameters": [ { "name": "attn_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if attention type is supported.", "summary": "Check if attention type is supported.", "implementation": "Method `MLXAttentionBackend.supports_attn_type` returns `attn_type in ['decoder', 'encoder', 'encoder_decoder']`.", "documented": true, "public": true, "addressable": true, "line": 133, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L133-L135", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "attn_type in ['decoder', 'encoder', 'encoder_decoder']" ], "awaits": false, "yields": false }, { "name": "MLXAttentionImpl", "qualname": "MLXAttentionImpl", "full_name": "vllm_mlx.attention.MLXAttentionImpl", "kind": "class", "signature": "class MLXAttentionImpl", "parameters": [ { "name": "num_heads", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of attention heads" }, { "name": "head_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Size of each head" }, { "name": "scale", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Attention scale factor" }, { "name": "num_kv_heads", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Number of KV heads (for GQA/MQA)" }, { "name": "alibi_slopes", "kind": "positional or keyword", "annotation": "list[float] | None", "default": "None", "required": false, "description": "ALiBi position encoding slopes" }, { "name": "sliding_window", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Sliding window attention size" }, { "name": "kv_cache_dtype", "kind": "positional or keyword", "annotation": "str", "default": "'auto'", "required": false, "description": "KV cache data type" }, { "name": "blocksparse_params", "kind": "positional or keyword", "annotation": "dict | None", "default": "None", "required": false, "description": "Block-sparse attention params" }, { "name": "logits_soft_cap", "kind": "positional or keyword", "annotation": "float | None", "default": "None", "required": false, "description": "Soft cap for logits" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "MLXAttentionImpl", "docstring": "MLX attention implementation.\n\nThis class provides the actual attention computation using MLX.\nSince mlx-lm handles attention internally during generation,\nthis serves as a compatibility interface.", "summary": "MLX attention implementation.", "implementation": "Class `MLXAttentionImpl` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 138, "end_line": 240, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L138-L240", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLXAttentionImpl.__init__", "full_name": "vllm_mlx.attention.MLXAttentionImpl.__init__", "kind": "method", "signature": "def __init__(self, num_heads: int, head_size: int, scale: float, num_kv_heads: int | None=None, alibi_slopes: list[float] | None=None, sliding_window: int | None=None, kv_cache_dtype: str='auto', blocksparse_params: dict | None=None, logits_soft_cap: float | None=None, **kwargs)", "parameters": [ { "name": "num_heads", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of attention heads" }, { "name": "head_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Size of each head" }, { "name": "scale", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Attention scale factor" }, { "name": "num_kv_heads", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Number of KV heads (for GQA/MQA)" }, { "name": "alibi_slopes", "kind": "positional or keyword", "annotation": "list[float] | None", "default": "None", "required": false, "description": "ALiBi position encoding slopes" }, { "name": "sliding_window", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Sliding window attention size" }, { "name": "kv_cache_dtype", "kind": "positional or keyword", "annotation": "str", "default": "'auto'", "required": false, "description": "KV cache data type" }, { "name": "blocksparse_params", "kind": "positional or keyword", "annotation": "dict | None", "default": "None", "required": false, "description": "Block-sparse attention params" }, { "name": "logits_soft_cap", "kind": "positional or keyword", "annotation": "float | None", "default": "None", "required": false, "description": "Soft cap for logits" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "not annotated", "docstring": "Initialize MLX attention.\n\nArgs:\n num_heads: Number of attention heads\n head_size: Size of each head\n scale: Attention scale factor\n num_kv_heads: Number of KV heads (for GQA/MQA)\n alibi_slopes: ALiBi position encoding slopes\n sliding_window: Sliding window attention size\n kv_cache_dtype: KV cache data type\n blocksparse_params: Block-sparse attention params\n logits_soft_cap: Soft cap for logits", "summary": "Initialize MLX attention.", "implementation": "Method `MLXAttentionImpl.__init__` updates `self.num_heads`, `self.head_size`, `self.scale`, `self.num_kv_heads`; calls `logger.debug`.", "documented": true, "public": false, "addressable": true, "line": 147, "end_line": 186, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L147-L186", "decorators": [], "calls": [ "logger.debug" ], "state_reads": [ "self.num_kv_heads" ], "state_writes": [ "self.num_heads", "self.head_size", "self.scale", "self.num_kv_heads", "self.alibi_slopes", "self.sliding_window", "self.kv_cache_dtype", "self.logits_soft_cap" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "forward", "qualname": "MLXAttentionImpl.forward", "full_name": "vllm_mlx.attention.MLXAttentionImpl.forward", "kind": "method", "signature": "def forward(self, query: Any, key: Any, value: Any, kv_cache: Any | None=None, attn_metadata: MLXAttentionMetadata | None=None, output: Any | None=None, **kwargs) -> Any", "parameters": [ { "name": "query", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Query tensor" }, { "name": "key", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Key tensor" }, { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Value tensor" }, { "name": "kv_cache", "kind": "positional or keyword", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional KV cache" }, { "name": "attn_metadata", "kind": "positional or keyword", "annotation": "MLXAttentionMetadata | None", "default": "None", "required": false, "description": "Attention metadata" }, { "name": "output", "kind": "positional or keyword", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional output buffer" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "Any", "docstring": "Compute attention.\n\nNote: In the MLX backend, attention is handled internally by mlx-lm\nduring the generation process. This method is provided for\ncompatibility but may not be called directly.\n\nArgs:\n query: Query tensor\n key: Key tensor\n value: Value tensor\n kv_cache: Optional KV cache\n attn_metadata: Attention metadata\n output: Optional output buffer\n\nReturns:\n Attention output tensor", "summary": "Compute attention.", "implementation": "Method `MLXAttentionImpl.forward` calls `isinstance`, `mx.array`, `hasattr`, `query.numpy`; returns `attn_output`.", "documented": true, "public": true, "addressable": true, "line": 188, "end_line": 240, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L188-L240", "decorators": [], "calls": [ "isinstance", "mx.array", "hasattr", "query.numpy", "key.numpy", "value.numpy", "mx.fast.scaled_dot_product_attention", "logger.error" ], "state_reads": [ "self.scale" ], "state_writes": [], "raises": [], "return_expressions": [ "attn_output" ], "awaits": false, "yields": false }, { "name": "create_mlx_attention_backend", "qualname": "create_mlx_attention_backend", "full_name": "vllm_mlx.attention.create_mlx_attention_backend", "kind": "function", "signature": "def create_mlx_attention_backend() -> type", "parameters": [], "return_annotation": "type", "docstring": "Factory function to create MLX attention backend.", "summary": "Factory function to create MLX attention backend.", "implementation": "Function `create_mlx_attention_backend` returns `MLXAttentionBackend`.", "documented": true, "public": true, "addressable": true, "line": 243, "end_line": 245, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/attention.py#L243-L245", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "MLXAttentionBackend" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.audio", "path": "vllm_mlx/audio/__init__.py", "page_path": "reference/api/vllm_mlx/audio/index.md", "docstring": "Audio support for vllm-mlx using mlx-audio.\n\nProvides:\n- STT (Speech-to-Text): Whisper, Parakeet\n- TTS (Text-to-Speech): Kokoro, Chatterbox, VibeVoice, VoxCPM\n- Audio Processing: SAM-Audio (voice separation)", "summary": "Audio support for vllm-mlx using mlx-audio.", "line_count": 25, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/__init__.py#L1-L25", "members": [ "__all__" ], "symbols": [] }, { "name": "vllm_mlx.audio.processor", "path": "vllm_mlx/audio/processor.py", "page_path": "reference/api/vllm_mlx/audio/processor.md", "docstring": "Audio processing using mlx-audio.\n\nSupports:\n- SAM-Audio: Text-guided source separation (isolate voice from background)\n- MossFormer2: Speech enhancement (noise removal)", "summary": "Audio processing using mlx-audio.", "line_count": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L1-L214", "members": [ "logger", "DEFAULT_SAM_MODEL", "SeparationResult", "AudioProcessor", "separate_voice" ], "symbols": [ { "name": "SeparationResult", "qualname": "SeparationResult", "full_name": "vllm_mlx.audio.processor.SeparationResult", "kind": "class", "signature": "class SeparationResult", "parameters": [ { "name": "target", "kind": "field", "annotation": "np.ndarray", "default": "", "required": true, "description": "Required constructor field." }, { "name": "residual", "kind": "field", "annotation": "np.ndarray", "default": "", "required": true, "description": "Required constructor field." }, { "name": "sample_rate", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "peak_memory", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "SeparationResult", "docstring": "Result from audio separation.", "summary": "Result from audio separation.", "implementation": "Class `SeparationResult` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 24, "end_line": 30, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L24-L30", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "AudioProcessor", "qualname": "AudioProcessor", "full_name": "vllm_mlx.audio.processor.AudioProcessor", "kind": "class", "signature": "class AudioProcessor", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_SAM_MODEL", "required": false, "description": "HuggingFace model name. Supported: - mlx-community/sam-audio-large-fp16 (best quality) - mlx-community/sam-audio-large - mlx-community/sam-audio-small-fp16 (faster) - mlx-community/sam-audio-small" } ], "return_annotation": "AudioProcessor", "docstring": "Audio processor for voice separation and enhancement.\n\nUses SAM-Audio for text-guided source separation:\n- Isolate speech from music/noise\n- Extract specific sounds by description\n\nUsage:\n processor = AudioProcessor()\n processor.load()\n result = processor.separate(\"meeting.mp3\", description=\"speech\")\n processor.save(result.target, \"voice_only.wav\")", "summary": "Audio processor for voice separation and enhancement.", "implementation": "Class `AudioProcessor` declares 6 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 33, "end_line": 192, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L33-L192", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "AudioProcessor.__init__", "full_name": "vllm_mlx.audio.processor.AudioProcessor.__init__", "kind": "method", "signature": "def __init__(self, model_name: str=DEFAULT_SAM_MODEL)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_SAM_MODEL", "required": false, "description": "HuggingFace model name. Supported: - mlx-community/sam-audio-large-fp16 (best quality) - mlx-community/sam-audio-large - mlx-community/sam-audio-small-fp16 (faster) - mlx-community/sam-audio-small" } ], "return_annotation": "not annotated", "docstring": "Initialize audio processor.\n\nArgs:\n model_name: HuggingFace model name. Supported:\n - mlx-community/sam-audio-large-fp16 (best quality)\n - mlx-community/sam-audio-large\n - mlx-community/sam-audio-small-fp16 (faster)\n - mlx-community/sam-audio-small", "summary": "Initialize audio processor.", "implementation": "Method `AudioProcessor.__init__` updates `self.model_name`, `self.model`, `self.processor`, `self._loaded`.", "documented": true, "public": false, "addressable": true, "line": 48, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L48-L66", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.model_name", "self.model", "self.processor", "self._loaded", "self.sample_rate" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load", "qualname": "AudioProcessor.load", "full_name": "vllm_mlx.audio.processor.AudioProcessor.load", "kind": "method", "signature": "def load(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the SAM-Audio model.", "summary": "Load the SAM-Audio model.", "implementation": "Method `AudioProcessor.load` updates `self.model`, `self.processor`, `self.sample_rate`, `self._loaded`; calls `SAMAudio.from_pretrained`, `SAMAudioProcessor.from_pretrained`, `hasattr`, `logger.info`; can raise `ImportError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 68, "end_line": 88, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L68-L88", "decorators": [], "calls": [ "SAMAudio.from_pretrained", "SAMAudioProcessor.from_pretrained", "hasattr", "logger.info", "logger.error", "ImportError" ], "state_reads": [ "self._loaded", "self.model_name", "self.model", "self.model.sample_rate" ], "state_writes": [ "self.model", "self.processor", "self.sample_rate", "self._loaded" ], "raises": [ "ImportError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "separate", "qualname": "AudioProcessor.separate", "full_name": "vllm_mlx.audio.processor.AudioProcessor.separate", "kind": "method", "signature": "def separate(self, audio_path: Union[str, Path], description: str='speech', chunk_seconds: Optional[float]=None) -> SeparationResult", "parameters": [ { "name": "audio_path", "kind": "positional or keyword", "annotation": "Union[str, Path]", "default": "", "required": true, "description": "Path to audio file" }, { "name": "description", "kind": "positional or keyword", "annotation": "str", "default": "'speech'", "required": false, "description": "What to isolate (e.g., \"speech\", \"music\", \"a person speaking\")" }, { "name": "chunk_seconds", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Process in chunks for long audio (memory efficient)" } ], "return_annotation": "SeparationResult", "docstring": "Separate audio based on text description.\n\nArgs:\n audio_path: Path to audio file\n description: What to isolate (e.g., \"speech\", \"music\", \"a person speaking\")\n chunk_seconds: Process in chunks for long audio (memory efficient)\n\nReturns:\n SeparationResult with target (isolated) and residual (background) audio", "summary": "Separate audio based on text description.", "implementation": "Method `AudioProcessor.separate` calls `self.load`, `str`, `self.processor`, `self.model.separate_long`; returns `SeparationResult(target=target, residual=residual, sample_rate=self.sample_rate, peak_memory=getattr(result, 'peak_memo…`.", "documented": true, "public": true, "addressable": true, "line": 90, "end_line": 151, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L90-L151", "decorators": [], "calls": [ "self.load", "str", "self.processor", "self.model.separate_long", "getattr", "self.model.separate", "self._to_numpy", "SeparationResult", "logger.error" ], "state_reads": [ "self._loaded", "self.load", "self.processor", "self.model.separate_long", "self.model", "self.model.separate", "self._to_numpy", "self.sample_rate" ], "state_writes": [], "raises": [], "return_expressions": [ "SeparationResult(target=target, residual=residual, sample_rate=self.sample_rate, peak_memory=getattr(result, 'peak_memo…" ], "awaits": false, "yields": false }, { "name": "_to_numpy", "qualname": "AudioProcessor._to_numpy", "full_name": "vllm_mlx.audio.processor.AudioProcessor._to_numpy", "kind": "method", "signature": "def _to_numpy(self, audio) -> np.ndarray", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "np.ndarray", "docstring": "Convert audio to numpy array.", "summary": "Convert audio to numpy array.", "implementation": "Method `AudioProcessor._to_numpy` calls `hasattr`, `np.array`, `audio.tolist`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 153, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L153-L157", "decorators": [], "calls": [ "hasattr", "np.array", "audio.tolist" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "np.array(audio.tolist(), dtype=np.float32)", "np.array(audio, dtype=np.float32)" ], "awaits": false, "yields": false }, { "name": "save", "qualname": "AudioProcessor.save", "full_name": "vllm_mlx.audio.processor.AudioProcessor.save", "kind": "method", "signature": "def save(self, audio: np.ndarray, path: Union[str, Path], sample_rate: Optional[int]=None) -> None", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "np.ndarray", "default": "", "required": true, "description": "Audio data as numpy array" }, { "name": "path", "kind": "positional or keyword", "annotation": "Union[str, Path]", "default": "", "required": true, "description": "Output file path" }, { "name": "sample_rate", "kind": "positional or keyword", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Sample rate (uses model default if None)" } ], "return_annotation": "None", "docstring": "Save audio to file.\n\nArgs:\n audio: Audio data as numpy array\n path: Output file path\n sample_rate: Sample rate (uses model default if None)", "summary": "Save audio to file.", "implementation": "Method `AudioProcessor.save` calls `save_audio`, `str`, `(audio * 32767).astype`, `wav.write`.", "documented": true, "public": true, "addressable": true, "line": 159, "end_line": 185, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L159-L185", "decorators": [], "calls": [ "save_audio", "str", "(audio * 32767).astype", "wav.write", "logger.info" ], "state_reads": [ "self.sample_rate" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "unload", "qualname": "AudioProcessor.unload", "full_name": "vllm_mlx.audio.processor.AudioProcessor.unload", "kind": "method", "signature": "def unload(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Unload model to free memory.", "summary": "Unload model to free memory.", "implementation": "Method `AudioProcessor.unload` updates `self.model`, `self.processor`, `self._loaded`; calls `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 187, "end_line": 192, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L187-L192", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [ "self.model", "self.processor", "self._loaded" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "separate_voice", "qualname": "separate_voice", "full_name": "vllm_mlx.audio.processor.separate_voice", "kind": "function", "signature": "def separate_voice(audio_path: Union[str, Path], model_name: str=DEFAULT_SAM_MODEL, description: str='speech') -> Tuple[np.ndarray, np.ndarray]", "parameters": [ { "name": "audio_path", "kind": "positional or keyword", "annotation": "Union[str, Path]", "default": "", "required": true, "description": "Path to audio file" }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_SAM_MODEL", "required": false, "description": "Model to use" }, { "name": "description", "kind": "positional or keyword", "annotation": "str", "default": "'speech'", "required": false, "description": "What to isolate" } ], "return_annotation": "Tuple[np.ndarray, np.ndarray]", "docstring": "Convenience function to separate voice from audio.\n\nArgs:\n audio_path: Path to audio file\n model_name: Model to use\n description: What to isolate\n\nReturns:\n Tuple of (voice_audio, background_audio) as numpy arrays", "summary": "Convenience function to separate voice from audio.", "implementation": "Function `separate_voice` calls `AudioProcessor`, `processor.load`, `processor.separate`; returns `(result.target, result.residual)`.", "documented": true, "public": true, "addressable": true, "line": 195, "end_line": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/processor.py#L195-L214", "decorators": [], "calls": [ "AudioProcessor", "processor.load", "processor.separate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(result.target, result.residual)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.audio.stt", "path": "vllm_mlx/audio/stt.py", "page_path": "reference/api/vllm_mlx/audio/stt.md", "docstring": "Speech-to-Text (STT) engine using mlx-audio.\n\nSupports:\n- Whisper (multilingual, 99+ languages)\n- Parakeet (English-focused, fast)", "summary": "Speech-to-Text (STT) engine using mlx-audio.", "line_count": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L1-L160", "members": [ "logger", "DEFAULT_WHISPER_MODEL", "DEFAULT_PARAKEET_MODEL", "TranscriptionResult", "STTEngine", "transcribe_audio" ], "symbols": [ { "name": "TranscriptionResult", "qualname": "TranscriptionResult", "full_name": "vllm_mlx.audio.stt.TranscriptionResult", "kind": "class", "signature": "class TranscriptionResult", "parameters": [ { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "language", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "duration", "kind": "field", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "segments", "kind": "field", "annotation": "Optional[list]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "TranscriptionResult", "docstring": "Result from audio transcription.", "summary": "Result from audio transcription.", "implementation": "Class `TranscriptionResult` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 23, "end_line": 29, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L23-L29", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "STTEngine", "qualname": "STTEngine", "full_name": "vllm_mlx.audio.stt.STTEngine", "kind": "class", "signature": "class STTEngine", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_WHISPER_MODEL", "required": false, "description": "HuggingFace model name. Supported: - mlx-community/whisper-large-v3-mlx (multilingual) - mlx-community/whisper-large-v3-turbo (fast) - mlx-community/whisper-medium-mlx - mlx-community/whisper-small-mlx - mlx-community/parakeet-tdt-0.6b-v2 (English, fastest) - mlx-community/parakeet-tdt-0.6b-v3" } ], "return_annotation": "STTEngine", "docstring": "Speech-to-Text engine supporting Whisper and Parakeet models.\n\nUsage:\n engine = STTEngine(\"mlx-community/whisper-large-v3-mlx\")\n engine.load()\n result = engine.transcribe(\"audio.mp3\")\n print(result.text)", "summary": "Speech-to-Text engine supporting Whisper and Parakeet models.", "implementation": "Class `STTEngine` declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 32, "end_line": 139, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L32-L139", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "STTEngine.__init__", "full_name": "vllm_mlx.audio.stt.STTEngine.__init__", "kind": "method", "signature": "def __init__(self, model_name: str=DEFAULT_WHISPER_MODEL)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_WHISPER_MODEL", "required": false, "description": "HuggingFace model name. Supported: - mlx-community/whisper-large-v3-mlx (multilingual) - mlx-community/whisper-large-v3-turbo (fast) - mlx-community/whisper-medium-mlx - mlx-community/whisper-small-mlx - mlx-community/parakeet-tdt-0.6b-v2 (English, fastest) - mlx-community/parakeet-tdt-0.6b-v3" } ], "return_annotation": "not annotated", "docstring": "Initialize STT engine.\n\nArgs:\n model_name: HuggingFace model name. Supported:\n - mlx-community/whisper-large-v3-mlx (multilingual)\n - mlx-community/whisper-large-v3-turbo (fast)\n - mlx-community/whisper-medium-mlx\n - mlx-community/whisper-small-mlx\n - mlx-community/parakeet-tdt-0.6b-v2 (English, fastest)\n - mlx-community/parakeet-tdt-0.6b-v3", "summary": "Initialize STT engine.", "implementation": "Method `STTEngine.__init__` updates `self.model_name`, `self.model`, `self._loaded`, `self._is_parakeet`; calls `model_name.lower`.", "documented": true, "public": false, "addressable": true, "line": 43, "end_line": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L43-L62", "decorators": [], "calls": [ "model_name.lower" ], "state_reads": [], "state_writes": [ "self.model_name", "self.model", "self._loaded", "self._is_parakeet" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load", "qualname": "STTEngine.load", "full_name": "vllm_mlx.audio.stt.STTEngine.load", "kind": "method", "signature": "def load(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the STT model.", "summary": "Load the STT model.", "implementation": "Method `STTEngine.load` updates `self.model`, `self._loaded`; calls `load_model`, `logger.info`, `logger.error`, `ImportError`; can raise `ImportError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 64, "end_line": 79, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L64-L79", "decorators": [], "calls": [ "load_model", "logger.info", "logger.error", "ImportError" ], "state_reads": [ "self._loaded", "self.model_name" ], "state_writes": [ "self.model", "self._loaded" ], "raises": [ "ImportError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "transcribe", "qualname": "STTEngine.transcribe", "full_name": "vllm_mlx.audio.stt.STTEngine.transcribe", "kind": "method", "signature": "def transcribe(self, audio_path: Union[str, Path], language: Optional[str]=None, task: str='transcribe') -> TranscriptionResult", "parameters": [ { "name": "audio_path", "kind": "positional or keyword", "annotation": "Union[str, Path]", "default": "", "required": true, "description": "Path to audio file (mp3, wav, m4a, etc.)" }, { "name": "language", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Language code (e.g., \"en\", \"es\"). Auto-detected if None." }, { "name": "task", "kind": "positional or keyword", "annotation": "str", "default": "'transcribe'", "required": false, "description": "\"transcribe\" or \"translate\" (translate to English)" } ], "return_annotation": "TranscriptionResult", "docstring": "Transcribe audio file to text.\n\nArgs:\n audio_path: Path to audio file (mp3, wav, m4a, etc.)\n language: Language code (e.g., \"en\", \"es\"). Auto-detected if None.\n task: \"transcribe\" or \"translate\" (translate to English)\n\nReturns:\n TranscriptionResult with text and metadata", "summary": "Transcribe audio file to text.", "implementation": "Method `STTEngine.transcribe` calls `self.load`, `str`, `self.model.generate`, `getattr`; returns `TranscriptionResult(text=text.strip() if isinstance(text, str) else str(text), language=detected_lang, duration=duratio…`.", "documented": true, "public": true, "addressable": true, "line": 81, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L81-L133", "decorators": [], "calls": [ "self.load", "str", "self.model.generate", "getattr", "hasattr", "TranscriptionResult", "isinstance", "text.strip", "logger.error" ], "state_reads": [ "self._loaded", "self.load", "self._is_parakeet", "self.model.generate", "self.model" ], "state_writes": [], "raises": [], "return_expressions": [ "TranscriptionResult(text=text.strip() if isinstance(text, str) else str(text), language=detected_lang, duration=duratio…" ], "awaits": false, "yields": false }, { "name": "unload", "qualname": "STTEngine.unload", "full_name": "vllm_mlx.audio.stt.STTEngine.unload", "kind": "method", "signature": "def unload(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Unload model to free memory.", "summary": "Unload model to free memory.", "implementation": "Method `STTEngine.unload` updates `self.model`, `self._loaded`; calls `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 135, "end_line": 139, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L135-L139", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [ "self.model", "self._loaded" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "transcribe_audio", "qualname": "transcribe_audio", "full_name": "vllm_mlx.audio.stt.transcribe_audio", "kind": "function", "signature": "def transcribe_audio(audio_path: Union[str, Path], model_name: str=DEFAULT_WHISPER_MODEL, language: Optional[str]=None) -> TranscriptionResult", "parameters": [ { "name": "audio_path", "kind": "positional or keyword", "annotation": "Union[str, Path]", "default": "", "required": true, "description": "Path to audio file" }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_WHISPER_MODEL", "required": false, "description": "Model to use" }, { "name": "language", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Language code (optional)" } ], "return_annotation": "TranscriptionResult", "docstring": "Convenience function to transcribe audio without managing engine.\n\nArgs:\n audio_path: Path to audio file\n model_name: Model to use\n language: Language code (optional)\n\nReturns:\n TranscriptionResult", "summary": "Convenience function to transcribe audio without managing engine.", "implementation": "Function `transcribe_audio` calls `STTEngine`, `engine.load`, `engine.transcribe`; returns `engine.transcribe(audio_path, language=language)`.", "documented": true, "public": true, "addressable": true, "line": 142, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/stt.py#L142-L160", "decorators": [], "calls": [ "STTEngine", "engine.load", "engine.transcribe" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "engine.transcribe(audio_path, language=language)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.audio.tts", "path": "vllm_mlx/audio/tts.py", "page_path": "reference/api/vllm_mlx/audio/tts.md", "docstring": "Text-to-Speech (TTS) engine using mlx-audio.\n\nSupports:\n- Kokoro (fast, lightweight)\n- Chatterbox (multilingual, expressive)\n- VibeVoice (realtime, low latency)\n- VoxCPM (Chinese/English, high quality)", "summary": "Text-to-Speech (TTS) engine using mlx-audio.", "line_count": 315, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L1-L315", "members": [ "logger", "DEFAULT_TTS_MODEL", "KOKORO_VOICES", "CHATTERBOX_VOICES", "AudioOutput", "TTSEngine", "generate_speech" ], "symbols": [ { "name": "AudioOutput", "qualname": "AudioOutput", "full_name": "vllm_mlx.audio.tts.AudioOutput", "kind": "class", "signature": "class AudioOutput", "parameters": [ { "name": "audio", "kind": "field", "annotation": "np.ndarray", "default": "", "required": true, "description": "Required constructor field." }, { "name": "sample_rate", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "duration", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "AudioOutput", "docstring": "Output from TTS generation.", "summary": "Output from TTS generation.", "implementation": "Class `AudioOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 44, "end_line": 49, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L44-L49", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "TTSEngine", "qualname": "TTSEngine", "full_name": "vllm_mlx.audio.tts.TTSEngine", "kind": "class", "signature": "class TTSEngine", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_TTS_MODEL", "required": false, "description": "HuggingFace model name. Supported families: - Kokoro: mlx-community/Kokoro-82M-bf16, Kokoro-82M-4bit - Chatterbox: mlx-community/chatterbox-turbo-fp16 - VibeVoice: mlx-community/VibeVoice-Realtime-0.5B-4bit - VoxCPM: mlx-community/VoxCPM1.5" } ], "return_annotation": "TTSEngine", "docstring": "Text-to-Speech engine supporting multiple model families.\n\nUsage:\n engine = TTSEngine(\"mlx-community/Kokoro-82M-bf16\")\n engine.load()\n audio = engine.generate(\"Hello world!\", voice=\"af_heart\")\n engine.save(audio, \"output.wav\")", "summary": "Text-to-Speech engine supporting multiple model families.", "implementation": "Class `TTSEngine` declares 9 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 52, "end_line": 292, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L52-L292", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "TTSEngine.__init__", "full_name": "vllm_mlx.audio.tts.TTSEngine.__init__", "kind": "method", "signature": "def __init__(self, model_name: str=DEFAULT_TTS_MODEL)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_TTS_MODEL", "required": false, "description": "HuggingFace model name. Supported families: - Kokoro: mlx-community/Kokoro-82M-bf16, Kokoro-82M-4bit - Chatterbox: mlx-community/chatterbox-turbo-fp16 - VibeVoice: mlx-community/VibeVoice-Realtime-0.5B-4bit - VoxCPM: mlx-community/VoxCPM1.5" } ], "return_annotation": "not annotated", "docstring": "Initialize TTS engine.\n\nArgs:\n model_name: HuggingFace model name. Supported families:\n - Kokoro: mlx-community/Kokoro-82M-bf16, Kokoro-82M-4bit\n - Chatterbox: mlx-community/chatterbox-turbo-fp16\n - VibeVoice: mlx-community/VibeVoice-Realtime-0.5B-4bit\n - VoxCPM: mlx-community/VoxCPM1.5", "summary": "Initialize TTS engine.", "implementation": "Method `TTSEngine.__init__` updates `self.model_name`, `self.model`, `self._loaded`, `self._model_family`; calls `self._detect_family`.", "documented": true, "public": false, "addressable": true, "line": 63, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L63-L80", "decorators": [], "calls": [ "self._detect_family" ], "state_reads": [ "self._detect_family" ], "state_writes": [ "self.model_name", "self.model", "self._loaded", "self._model_family" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_detect_family", "qualname": "TTSEngine._detect_family", "full_name": "vllm_mlx.audio.tts.TTSEngine._detect_family", "kind": "method", "signature": "def _detect_family(self, model_name: str) -> str", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Detect model family from name.", "summary": "Detect model family from name.", "implementation": "Method `TTSEngine._detect_family` calls `model_name.lower`; has 6 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 82, "end_line": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L82-L98", "decorators": [], "calls": [ "model_name.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'kokoro'", "'chatterbox'", "'vibevoice'", "'voxcpm'", "'csm'", "'cosyvoice'" ], "awaits": false, "yields": false }, { "name": "load", "qualname": "TTSEngine.load", "full_name": "vllm_mlx.audio.tts.TTSEngine.load", "kind": "method", "signature": "def load(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the TTS model.", "summary": "Load the TTS model.", "implementation": "Method `TTSEngine.load` updates `self.model`, `self._loaded`; calls `load_model`, `logger.info`, `logger.error`, `ImportError`; can raise `ImportError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 100, "end_line": 117, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L100-L117", "decorators": [], "calls": [ "load_model", "logger.info", "logger.error", "ImportError" ], "state_reads": [ "self._loaded", "self.model_name", "self._model_family" ], "state_writes": [ "self.model", "self._loaded" ], "raises": [ "ImportError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "generate", "qualname": "TTSEngine.generate", "full_name": "vllm_mlx.audio.tts.TTSEngine.generate", "kind": "method", "signature": "def generate(self, text: str, voice: str='af_heart', speed: float=1.0, lang_code: str='a') -> AudioOutput", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text to synthesize" }, { "name": "voice", "kind": "positional or keyword", "annotation": "str", "default": "'af_heart'", "required": false, "description": "Voice ID (model-specific)" }, { "name": "speed", "kind": "positional or keyword", "annotation": "float", "default": "1.0", "required": false, "description": "Speech speed (0.5 to 2.0)" }, { "name": "lang_code", "kind": "positional or keyword", "annotation": "str", "default": "'a'", "required": false, "description": "Language code (a=English, e=Spanish, f=French, etc.)" } ], "return_annotation": "AudioOutput", "docstring": "Generate speech from text.\n\nArgs:\n text: Text to synthesize\n voice: Voice ID (model-specific)\n speed: Speech speed (0.5 to 2.0)\n lang_code: Language code (a=English, e=Spanish, f=French, etc.)\n\nReturns:\n AudioOutput with audio data and metadata", "summary": "Generate speech from text.", "implementation": "Method `TTSEngine.generate` calls `self.load`, `self.model.generate`, `hasattr`, `isinstance`; can raise `RuntimeError`; returns `AudioOutput(audio=full_audio, sample_rate=sample_rate, duration=duration)`.", "documented": true, "public": true, "addressable": true, "line": 119, "end_line": 185, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L119-L185", "decorators": [], "calls": [ "self.load", "self.model.generate", "hasattr", "isinstance", "np.array", "audio_data.tolist", "audio_chunks.append", "RuntimeError", "len", "np.concatenate", "AudioOutput", "logger.error" ], "state_reads": [ "self._loaded", "self.load", "self.model.generate", "self.model" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "AudioOutput(audio=full_audio, sample_rate=sample_rate, duration=duration)" ], "awaits": false, "yields": false }, { "name": "stream_generate", "qualname": "TTSEngine.stream_generate", "full_name": "vllm_mlx.audio.tts.TTSEngine.stream_generate", "kind": "method", "signature": "def stream_generate(self, text: str, voice: str='af_heart', speed: float=1.0) -> Iterator[AudioOutput]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text to synthesize" }, { "name": "voice", "kind": "positional or keyword", "annotation": "str", "default": "'af_heart'", "required": false, "description": "Voice ID" }, { "name": "speed", "kind": "positional or keyword", "annotation": "float", "default": "1.0", "required": false, "description": "Speech speed" } ], "return_annotation": "Iterator[AudioOutput]", "docstring": "Stream speech generation chunk by chunk.\n\nArgs:\n text: Text to synthesize\n voice: Voice ID\n speed: Speech speed\n\nYields:\n AudioOutput chunks", "summary": "Stream speech generation chunk by chunk.", "implementation": "Method `TTSEngine.stream_generate` calls `self.load`, `self.model.generate`, `hasattr`, `np.array`; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 187, "end_line": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L187-L227", "decorators": [], "calls": [ "self.load", "self.model.generate", "hasattr", "np.array", "audio_data.tolist", "AudioOutput", "len" ], "state_reads": [ "self._loaded", "self.load", "self.model.generate", "self.model" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "save", "qualname": "TTSEngine.save", "full_name": "vllm_mlx.audio.tts.TTSEngine.save", "kind": "method", "signature": "def save(self, audio: AudioOutput, path: Union[str, Path], format: str='wav') -> None", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "AudioOutput", "default": "", "required": true, "description": "AudioOutput to save" }, { "name": "path", "kind": "positional or keyword", "annotation": "Union[str, Path]", "default": "", "required": true, "description": "Output file path" }, { "name": "format", "kind": "positional or keyword", "annotation": "str", "default": "'wav'", "required": false, "description": "Output format (wav, mp3)" } ], "return_annotation": "None", "docstring": "Save audio to file.\n\nArgs:\n audio: AudioOutput to save\n path: Output file path\n format: Output format (wav, mp3)", "summary": "Save audio to file.", "implementation": "Method `TTSEngine.save` calls `save_audio`, `str`, `logger.info`, `(audio.audio * 32767).astype`.", "documented": true, "public": true, "addressable": true, "line": 229, "end_line": 255, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L229-L255", "decorators": [], "calls": [ "save_audio", "str", "logger.info", "(audio.audio * 32767).astype", "wav.write" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_bytes", "qualname": "TTSEngine.to_bytes", "full_name": "vllm_mlx.audio.tts.TTSEngine.to_bytes", "kind": "method", "signature": "def to_bytes(self, audio: AudioOutput, format: str='wav') -> bytes", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "AudioOutput", "default": "", "required": true, "description": "AudioOutput to convert" }, { "name": "format", "kind": "positional or keyword", "annotation": "str", "default": "'wav'", "required": false, "description": "Output format (wav, mp3)" } ], "return_annotation": "bytes", "docstring": "Convert audio to bytes.\n\nArgs:\n audio: AudioOutput to convert\n format: Output format (wav, mp3)\n\nReturns:\n Audio data as bytes", "summary": "Convert audio to bytes.", "implementation": "Method `TTSEngine.to_bytes` calls `io.BytesIO`, `(audio.audio * 32767).astype`, `wav.write`, `buffer.getvalue`; returns `buffer.getvalue()`.", "documented": true, "public": true, "addressable": true, "line": 257, "end_line": 277, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L257-L277", "decorators": [], "calls": [ "io.BytesIO", "(audio.audio * 32767).astype", "wav.write", "buffer.getvalue" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "buffer.getvalue()" ], "awaits": false, "yields": false }, { "name": "get_voices", "qualname": "TTSEngine.get_voices", "full_name": "vllm_mlx.audio.tts.TTSEngine.get_voices", "kind": "method", "signature": "def get_voices(self) -> list", "parameters": [], "return_annotation": "list", "docstring": "Get available voices for current model.", "summary": "Get available voices for current model.", "implementation": "Method `TTSEngine.get_voices` has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 279, "end_line": 286, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L279-L286", "decorators": [], "calls": [], "state_reads": [ "self._model_family" ], "state_writes": [], "raises": [], "return_expressions": [ "KOKORO_VOICES", "CHATTERBOX_VOICES", "['default']" ], "awaits": false, "yields": false }, { "name": "unload", "qualname": "TTSEngine.unload", "full_name": "vllm_mlx.audio.tts.TTSEngine.unload", "kind": "method", "signature": "def unload(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Unload model to free memory.", "summary": "Unload model to free memory.", "implementation": "Method `TTSEngine.unload` updates `self.model`, `self._loaded`; calls `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 288, "end_line": 292, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L288-L292", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [ "self.model", "self._loaded" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "generate_speech", "qualname": "generate_speech", "full_name": "vllm_mlx.audio.tts.generate_speech", "kind": "function", "signature": "def generate_speech(text: str, model_name: str=DEFAULT_TTS_MODEL, voice: str='af_heart', speed: float=1.0) -> AudioOutput", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text to synthesize" }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "DEFAULT_TTS_MODEL", "required": false, "description": "Model to use" }, { "name": "voice", "kind": "positional or keyword", "annotation": "str", "default": "'af_heart'", "required": false, "description": "Voice ID" }, { "name": "speed", "kind": "positional or keyword", "annotation": "float", "default": "1.0", "required": false, "description": "Speech speed" } ], "return_annotation": "AudioOutput", "docstring": "Convenience function to generate speech without managing engine.\n\nArgs:\n text: Text to synthesize\n model_name: Model to use\n voice: Voice ID\n speed: Speech speed\n\nReturns:\n AudioOutput", "summary": "Convenience function to generate speech without managing engine.", "implementation": "Function `generate_speech` calls `TTSEngine`, `engine.load`, `engine.generate`; returns `engine.generate(text, voice=voice, speed=speed)`.", "documented": true, "public": true, "addressable": true, "line": 295, "end_line": 315, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio/tts.py#L295-L315", "decorators": [], "calls": [ "TTSEngine", "engine.load", "engine.generate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "engine.generate(text, voice=voice, speed=speed)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.audio_limits", "path": "vllm_mlx/audio_limits.py", "page_path": "reference/api/vllm_mlx/audio_limits.md", "docstring": "Resource limits for optional audio endpoints.", "summary": "Resource limits for optional audio endpoints.", "line_count": 77, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio_limits.py#L1-L77", "members": [ "DEFAULT_MAX_AUDIO_UPLOAD_MB", "DEFAULT_MAX_AUDIO_UPLOAD_BYTES", "DEFAULT_MAX_TTS_INPUT_CHARS", "UPLOAD_CHUNK_SIZE", "AsyncReadableUpload", "save_upload_with_limit", "validate_tts_input_length" ], "symbols": [ { "name": "AsyncReadableUpload", "qualname": "AsyncReadableUpload", "full_name": "vllm_mlx.audio_limits.AsyncReadableUpload", "kind": "class", "signature": "class AsyncReadableUpload(Protocol)", "parameters": [], "return_annotation": "AsyncReadableUpload", "docstring": "Structural type for an asynchronously readable uploaded file.", "summary": "Structural type for an asynchronously readable uploaded file.", "implementation": "Class `AsyncReadableUpload` derives from `Protocol` and declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 17, "end_line": 25, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio_limits.py#L17-L25", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "read", "qualname": "AsyncReadableUpload.read", "full_name": "vllm_mlx.audio_limits.AsyncReadableUpload.read", "kind": "method", "signature": "async def read(self, size: int=-1) -> bytes", "parameters": [ { "name": "size", "kind": "positional or keyword", "annotation": "int", "default": "-1", "required": false, "description": "Optional positional or keyword input; defaults to `-1`." } ], "return_annotation": "bytes", "docstring": "Read at most ``size`` bytes, or all remaining bytes when negative.", "summary": "Read at most ``size`` bytes, or all remaining bytes when negative.", "implementation": "Method `AsyncReadableUpload.read` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 25, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio_limits.py#L22-L25", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "save_upload_with_limit", "qualname": "save_upload_with_limit", "full_name": "vllm_mlx.audio_limits.save_upload_with_limit", "kind": "function", "signature": "async def save_upload_with_limit(file: AsyncReadableUpload, *, max_bytes: int, default_suffix: str='.wav', chunk_size: int=UPLOAD_CHUNK_SIZE) -> str", "parameters": [ { "name": "file", "kind": "positional or keyword", "annotation": "AsyncReadableUpload", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_bytes", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "default_suffix", "kind": "keyword-only", "annotation": "str", "default": "'.wav'", "required": false, "description": "Optional keyword-only input; defaults to `'.wav'`." }, { "name": "chunk_size", "kind": "keyword-only", "annotation": "int", "default": "UPLOAD_CHUNK_SIZE", "required": false, "description": "Optional keyword-only input; defaults to `UPLOAD_CHUNK_SIZE`." } ], "return_annotation": "str", "docstring": "Stream an uploaded file to disk while enforcing a hard byte limit.\n\nThis prevents large audio uploads from being buffered entirely in memory.", "summary": "Stream an uploaded file to disk while enforcing a hard byte limit.", "implementation": "Function `save_upload_with_limit` calls `Path`, `tempfile.NamedTemporaryFile`, `file.read`, `len`; awaits asynchronous work; can raise `HTTPException`; returns `tmp_path`.", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio_limits.py#L28-L65", "decorators": [], "calls": [ "Path", "tempfile.NamedTemporaryFile", "file.read", "len", "HTTPException", "tmp.write", "os.path.exists", "os.unlink" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "tmp_path" ], "awaits": true, "yields": false }, { "name": "validate_tts_input_length", "qualname": "validate_tts_input_length", "full_name": "vllm_mlx.audio_limits.validate_tts_input_length", "kind": "function", "signature": "def validate_tts_input_length(text: str, *, max_chars: int) -> None", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_chars", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Reject oversized TTS requests before synthesis starts.", "summary": "Reject oversized TTS requests before synthesis starts.", "implementation": "Function `validate_tts_input_length` calls `len`, `HTTPException`; can raise `HTTPException`.", "documented": true, "public": true, "addressable": true, "line": 68, "end_line": 77, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/audio_limits.py#L68-L77", "decorators": [], "calls": [ "len", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.bench_serve", "path": "vllm_mlx/bench_serve.py", "page_path": "reference/api/vllm_mlx/bench_serve.md", "docstring": "Serving benchmark for vllm-mlx.\n\nMeasures end-to-end HTTP performance of a running vllm-mlx server:\n- Time to First Token (TTFT)\n- Time Per Output Token (TPOT)\n- End-to-end latency\n- Generation and prompt throughput\n- Concurrent request handling\n- KV cache hit rates\n- Metal memory utilization\n\nThis module has no MLX dependency and can be imported on any platform.\nIt is a pure HTTP client that talks to a running OpenAI-compatible server.", "summary": "Serving benchmark for vllm-mlx.", "line_count": 2638, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1-L2638", "members": [ "_BUILTIN_DIR", "_BUILTIN_NAMES", "_SQL_IDENTIFIER_RE", "WorkloadCase", "Workload", "load_prompt_set", "_require_message_list", "_load_case_request", "_request_extra_body", "_first_not_none", "_normalize_tags", "_merge_case_checks", "_build_workload_case", "load_workload", "BenchServeResult", "SweepConfig", "expand_sweep", "parse_health_response", "parse_status_response", "parse_metrics_text", "detect_hardware_fingerprint", "auto_detect_runtime", "scrape_metrics", "clear_runtime_cache", "_normalize_cache_policy", "parse_sse_line", "_cancel_server_request", "accumulate_tool_calls", "finalize_tool_calls", "compute_request_metrics", "count_prompt_tokens", "stream_chat_completion", "validate_response", "_check_finish_reason", "_check_length_bounds", "_check_regex_patterns", "_check_json_content", "_check_tool_call_count_and_names", "_check_tool_call_args", "validate_quality_checks", "compute_summary_stats", "run_concurrent_requests", "_summary_or_empty", "_resolve_max_tokens", "_assemble_case_request_kwargs", "_empty_completion_result", "_fetch_post_run_status", "_compute_within_policy_timeout", "_build_tool_calls_summary", "_build_workload_record", "run_workload_case", "_group_results_by_case_id", "_summarize_case", "summarize_workload_results", "run_bench_serve_workload", "RESULT_COLUMNS", "_TABLE_COLUMNS", "_result_to_dict", "format_table", "format_json", "format_csv", "_sql_escape", "_SQL_SCHEMA", "format_sql", "_write_sqlite_rows", "_validate_sql_identifier", "write_sqlite", "WORKLOAD_RESULT_COLUMNS", "_WORKLOAD_TABLE_COLUMNS", "_workload_record_to_row", "format_workload_table", "format_workload_json", "format_workload_csv", "_WORKLOAD_SQL_SCHEMA", "format_workload_sql", "write_workload_sqlite", "format_workload_payload", "logger", "run_bench_serve" ], "symbols": [ { "name": "WorkloadCase", "qualname": "WorkloadCase", "full_name": "vllm_mlx.bench_serve.WorkloadCase", "kind": "class", "signature": "class WorkloadCase", "parameters": [ { "name": "case_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "messages", "kind": "field", "annotation": "list[dict]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "request_path", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_tokens", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "enable_thinking", "kind": "field", "annotation": "Optional[bool]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "extra_body", "kind": "field", "annotation": "Optional[dict]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "policy_timeout_ms", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "checks", "kind": "field", "annotation": "Optional[dict]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tags", "kind": "field", "annotation": "tuple[str, ...]", "default": "()", "required": false, "description": "Optional constructor field; defaults to `()`." } ], "return_annotation": "WorkloadCase", "docstring": "One declarative benchmark case for contract-style serving tests.", "summary": "One declarative benchmark case for contract-style serving tests.", "implementation": "Class `WorkloadCase` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 51, "end_line": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L51-L62", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "Workload", "qualname": "Workload", "full_name": "vllm_mlx.bench_serve.Workload", "kind": "class", "signature": "class Workload", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "defaults", "kind": "field", "annotation": "dict", "default": "", "required": true, "description": "Required constructor field." }, { "name": "cases", "kind": "field", "annotation": "list[WorkloadCase]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "Workload", "docstring": "Normalized bench-serve workload manifest.", "summary": "Normalized bench-serve workload manifest.", "implementation": "Class `Workload` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 66, "end_line": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L66-L72", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load_prompt_set", "qualname": "load_prompt_set", "full_name": "vllm_mlx.bench_serve.load_prompt_set", "kind": "function", "signature": "def load_prompt_set(name_or_path: str) -> list[list[dict]]", "parameters": [ { "name": "name_or_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[list[dict]]", "docstring": "Load a prompt set by builtin name or file path.\n\nBuiltin sets (``short``, ``medium``, ``long``, ``thinking``) are loaded\nfrom the ``bench_serve_prompts/`` directory next to this module. Any\nother value is treated as a filesystem path and loaded directly.\n\nTwo file formats are accepted (detected automatically):\n\n1. **Flat** — list of single message dicts. Each dict becomes a\n single-message prompt. Backwards-compatible with the original format.\n\n ``[{\"role\": \"user\", \"content\": \"...\"}, ...]``\n\n2. **Multi-message** — list of message-dict lists. Each inner list is a\n full chat history (e.g. ``[system, user]``). Use this format when you\n want to benchmark with system prompts that match an ``--warm-prompts``\n warm-up, or to simulate multi-turn conversation.\n\n ``[[{\"role\":\"system\",\"content\":\"...\"}, {\"role\":\"user\",\"content\":\"...\"}], ...]``\n\nReturns:\n A list of message-dict lists, i.e. every entry is a full chat history.\n Flat-format files are normalized to single-element lists.\n\nRaises:\n FileNotFoundError: If ``name_or_path`` is not a known builtin name and\n the path does not exist, or if a builtin name is requested but its\n JSON file is missing from the package.\n ValueError: If the file shape is not recognised.", "summary": "Load a prompt set by builtin name or file path.", "implementation": "Function `load_prompt_set` calls `target.exists`, `FileNotFoundError`, `target.open`, `json.load`; can raise `FileNotFoundError`, `ValueError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 75, "end_line": 137, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L75-L137", "decorators": [], "calls": [ "target.exists", "FileNotFoundError", "target.open", "json.load", "Path(name_or_path).expanduser", "Path", "path.exists", "sorted", "path.open", "isinstance", "ValueError", "type" ], "state_reads": [], "state_writes": [], "raises": [ "FileNotFoundError", "ValueError" ], "return_expressions": [ "[[msg] for msg in raw]", "raw" ], "awaits": false, "yields": false }, { "name": "_require_message_list", "qualname": "_require_message_list", "full_name": "vllm_mlx.bench_serve._require_message_list", "kind": "function", "signature": "def _require_message_list(value: Any, *, label: str) -> list[dict]", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "label", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "list[dict]", "docstring": "", "summary": "Function `_require_message_list` calls `isinstance`, `ValueError`, `enumerate`; can raise `ValueError`; returns `value`.", "implementation": "Function `_require_message_list` calls `isinstance`, `ValueError`, `enumerate`; can raise `ValueError`; returns `value`.", "documented": false, "public": false, "addressable": true, "line": 140, "end_line": 148, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L140-L148", "decorators": [], "calls": [ "isinstance", "ValueError", "enumerate" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "value" ], "awaits": false, "yields": false }, { "name": "_load_case_request", "qualname": "_load_case_request", "full_name": "vllm_mlx.bench_serve._load_case_request", "kind": "function", "signature": "def _load_case_request(path: str, *, workload_path: Path, case_id: str) -> dict", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "workload_path", "kind": "keyword-only", "annotation": "Path", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "case_id", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "dict", "docstring": "", "summary": "Function `_load_case_request` calls `Path(path).expanduser`, `Path`, `request_path.is_absolute`, `request_path.open`; can raise `ValueError`; returns `request`.", "implementation": "Function `_load_case_request` calls `Path(path).expanduser`, `Path`, `request_path.is_absolute`, `request_path.open`; can raise `ValueError`; returns `request`.", "documented": false, "public": false, "addressable": true, "line": 151, "end_line": 159, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L151-L159", "decorators": [], "calls": [ "Path(path).expanduser", "Path", "request_path.is_absolute", "request_path.open", "json.load", "isinstance", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "request" ], "awaits": false, "yields": false }, { "name": "_request_extra_body", "qualname": "_request_extra_body", "full_name": "vllm_mlx.bench_serve._request_extra_body", "kind": "function", "signature": "def _request_extra_body(request: dict) -> dict", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "", "summary": "Function `_request_extra_body` calls `request.items`; returns `{key: value for key, value in request.items() if key not in reserved}`.", "implementation": "Function `_request_extra_body` calls `request.items`; returns `{key: value for key, value in request.items() if key not in reserved}`.", "documented": false, "public": false, "addressable": true, "line": 162, "end_line": 171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L162-L171", "decorators": [], "calls": [ "request.items" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{key: value for key, value in request.items() if key not in reserved}" ], "awaits": false, "yields": false }, { "name": "_first_not_none", "qualname": "_first_not_none", "full_name": "vllm_mlx.bench_serve._first_not_none", "kind": "function", "signature": "def _first_not_none(*values: Any) -> Any", "parameters": [ { "name": "*values", "kind": "variadic positional", "annotation": "Any", "default": "", "required": false, "description": "Additional variadic positional inputs accepted by this callable." } ], "return_annotation": "Any", "docstring": "", "summary": "Function `_first_not_none` has 2 explicit return paths.", "implementation": "Function `_first_not_none` has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 174, "end_line": 178, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L174-L178", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "value", "None" ], "awaits": false, "yields": false }, { "name": "_normalize_tags", "qualname": "_normalize_tags", "full_name": "vllm_mlx.bench_serve._normalize_tags", "kind": "function", "signature": "def _normalize_tags(tags: Any, *, case_id: str) -> tuple[str, ...]", "parameters": [ { "name": "tags", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "case_id", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "tuple[str, ...]", "docstring": "Coerce a workload case's ``tags`` field to a tuple of strings.\n\nAccepts either a single string (treated as a one-element list) or a\nlist. Any other type is rejected with a case-scoped ``ValueError``.", "summary": "Coerce a workload case's ``tags`` field to a tuple of strings.", "implementation": "Function `_normalize_tags` calls `isinstance`, `ValueError`, `tuple`, `str`; can raise `ValueError`; returns `tuple((str(tag) for tag in tags))`.", "documented": true, "public": false, "addressable": true, "line": 181, "end_line": 191, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L181-L191", "decorators": [], "calls": [ "isinstance", "ValueError", "tuple", "str" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "tuple((str(tag) for tag in tags))" ], "awaits": false, "yields": false }, { "name": "_merge_case_checks", "qualname": "_merge_case_checks", "full_name": "vllm_mlx.bench_serve._merge_case_checks", "kind": "function", "signature": "def _merge_case_checks(default_checks: Any, case_checks: Any, *, case_id: str) -> Optional[dict]", "parameters": [ { "name": "default_checks", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "case_checks", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "case_id", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "Optional[dict]", "docstring": "Merge a case's ``checks`` over the workload defaults.\n\nMost keys are overridden by the case-level value. The two regex list\nkeys (``required_regex``, ``forbidden_regex``) are list-concatenated\nwith default patterns first and case patterns appended, so\ncase-level patterns extend defaults rather than replace them.\nReturns ``None`` when neither source contributes any checks.\n\nRejects a non-dict ``case_checks`` with a ``ValueError`` named after\nthe case so the operator gets a clear message instead of the\n``AttributeError`` that the previous inline code raised on\n``case_checks.items()``.", "summary": "Merge a case's ``checks`` over the workload defaults.", "implementation": "Function `_merge_case_checks` calls `dict`, `isinstance`, `ValueError`, `case_checks.items`; can raise `ValueError`; returns `merged or None`.", "documented": true, "public": false, "addressable": true, "line": 194, "end_line": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L194-L227", "decorators": [], "calls": [ "dict", "isinstance", "ValueError", "case_checks.items", "merged.get" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "merged or None" ], "awaits": false, "yields": false }, { "name": "_build_workload_case", "qualname": "_build_workload_case", "full_name": "vllm_mlx.bench_serve._build_workload_case", "kind": "function", "signature": "def _build_workload_case(item: Any, idx: int, *, defaults: dict, workload_path: Path) -> WorkloadCase", "parameters": [ { "name": "item", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "defaults", "kind": "keyword-only", "annotation": "dict", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "workload_path", "kind": "keyword-only", "annotation": "Path", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "WorkloadCase", "docstring": "Construct one ``WorkloadCase`` from a raw workload entry.\n\nValidates the entry shape, loads request defaults from a sibling JSON\nfile when ``request_path`` is provided, merges ``extra_body`` and\n``checks`` against the workload defaults, and resolves scalar fields\n(``max_tokens``, ``enable_thinking``, ``policy_timeout_ms``) via\n``_first_not_none`` priority: case-level beats request_path defaults\nbeats workload defaults.\n\n``extra_body`` follows a different merge: it composes the\n``request_path`` extras (base) with either the case-level\n``extra_body`` if present, otherwise the workload-default\n``extra_body``. The case-vs-default fallback is a get-with-default,\nnot a three-way merge.", "summary": "Construct one ``WorkloadCase`` from a raw workload entry.", "implementation": "Function `_build_workload_case` calls `isinstance`, `ValueError`, `str`, `item.get`; can raise `ValueError`; returns `WorkloadCase(case_id=case_id, messages=messages, request_path=str(request_path) if request_path is not None else None, …`.", "documented": true, "public": false, "addressable": true, "line": 230, "end_line": 300, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L230-L300", "decorators": [], "calls": [ "isinstance", "ValueError", "str", "item.get", "_load_case_request", "_require_message_list", "request_defaults.get", "defaults.get", "_request_extra_body", "request_extra.update", "_merge_case_checks", "WorkloadCase", "_first_not_none", "_normalize_tags" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "WorkloadCase(case_id=case_id, messages=messages, request_path=str(request_path) if request_path is not None else None, …" ], "awaits": false, "yields": false }, { "name": "load_workload", "qualname": "load_workload", "full_name": "vllm_mlx.bench_serve.load_workload", "kind": "function", "signature": "def load_workload(path: str | Path) -> Workload", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "str | Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Workload", "docstring": "Load a declarative serving benchmark workload.\n\nWorkloads are for product-like qualification where each case can carry\nrequest settings, comparison-only policy timeouts, and quality checks.\nTimeout fields are metadata unless the runner explicitly uses them as a\ntransport limit; they are not treated as hardware capability claims.", "summary": "Load a declarative serving benchmark workload.", "implementation": "Function `load_workload` calls `Path(path).expanduser`, `Path`, `workload_path.open`, `json.load`; can raise `ValueError`; returns `Workload(name=str(raw.get('name') or workload_path.stem), description=str(raw.get('description') or ''), defaults=defau…`.", "documented": true, "public": true, "addressable": true, "line": 303, "end_line": 335, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L303-L335", "decorators": [], "calls": [ "Path(path).expanduser", "Path", "workload_path.open", "json.load", "isinstance", "ValueError", "raw.get", "_build_workload_case", "enumerate", "Workload", "str" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "Workload(name=str(raw.get('name') or workload_path.stem), description=str(raw.get('description') or ''), defaults=defau…" ], "awaits": false, "yields": false }, { "name": "BenchServeResult", "qualname": "BenchServeResult", "full_name": "vllm_mlx.bench_serve.BenchServeResult", "kind": "class", "signature": "class BenchServeResult", "parameters": [ { "name": "run_id", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "timestamp", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "tag", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "chip", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "gpu_cores", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "bandwidth_gbs", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "os_version", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "model_id", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "model_type", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "engine_type", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "mtp_enabled", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "specprefill", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "kv_quant", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "cache_type", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "prompt_set", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "concurrency", "kind": "field", "annotation": "int", "default": "1", "required": false, "description": "Optional constructor field; defaults to `1`." }, { "name": "max_tokens", "kind": "field", "annotation": "int", "default": "256", "required": false, "description": "Optional constructor field; defaults to `256`." }, { "name": "enable_thinking", "kind": "field", "annotation": "Optional[bool]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "extra_body", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "repetition", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "ttft_ms", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "tpot_ms", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "e2e_latency_ms", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "gen_tps", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "prompt_tps", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "throughput_tps", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "requests_per_s", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "metal_active_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "metal_peak_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "metal_cache_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "cache_hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "cache_misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "cache_hit_rate", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "tokens_saved", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "validated", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." } ], "return_annotation": "BenchServeResult", "docstring": "Aggregated results from a single bench-serve run configuration.", "summary": "Aggregated results from a single bench-serve run configuration.", "implementation": "Class `BenchServeResult` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 344, "end_line": 400, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L344-L400", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "expand_sweep", "qualname": "expand_sweep", "full_name": "vllm_mlx.bench_serve.expand_sweep", "kind": "function", "signature": "def expand_sweep(prompt_sets: list[str], concurrencies: list[int], thinking_values: list[Optional[bool]], extra_bodies: list[str], repetitions: int) -> list[SweepConfig]", "parameters": [ { "name": "prompt_sets", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Names or paths of prompt sets to include." }, { "name": "concurrencies", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Concurrency levels to test (e.g. ``[1, 4, 16]``)." }, { "name": "thinking_values", "kind": "positional or keyword", "annotation": "list[Optional[bool]]", "default": "", "required": true, "description": "Values for ``enable_thinking`` (e.g. ``[None, True, False]``)." }, { "name": "extra_bodies", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "JSON strings (or empty string) to pass as extra body parameters on each request." }, { "name": "repetitions", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of times to repeat each unique combination. Each repeat gets a distinct 0-based repetition index." } ], "return_annotation": "list[SweepConfig]", "docstring": "Expand sweep parameters into a flat list of configurations.\n\nPerforms the full Cartesian product of all input dimensions and then\nunfolds each combination across ``repetitions`` repetition indices\n(0-based).\n\nArgs:\n prompt_sets: Names or paths of prompt sets to include.\n concurrencies: Concurrency levels to test (e.g. ``[1, 4, 16]``).\n thinking_values: Values for ``enable_thinking`` (e.g.\n ``[None, True, False]``).\n extra_bodies: JSON strings (or empty string) to pass as extra body\n parameters on each request.\n repetitions: Number of times to repeat each unique combination.\n Each repeat gets a distinct 0-based repetition index.\n\nReturns:\n A list of :data:`SweepConfig` tuples in the order::\n\n (prompt_set, concurrency, thinking, extra_body, repetition_index)", "summary": "Expand sweep parameters into a flat list of configurations.", "implementation": "Function `expand_sweep` calls `itertools.product`, `range`, `configs.append`; returns `configs`.", "documented": true, "public": true, "addressable": true, "line": 411, "end_line": 444, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L411-L444", "decorators": [], "calls": [ "itertools.product", "range", "configs.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "configs" ], "awaits": false, "yields": false }, { "name": "parse_health_response", "qualname": "parse_health_response", "full_name": "vllm_mlx.bench_serve.parse_health_response", "kind": "function", "signature": "def parse_health_response(data: dict) -> dict", "parameters": [ { "name": "data", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Parsed JSON body from the /health endpoint. Expected shape:: {\"status\": \"healthy\", \"model_loaded\": True, \"model_name\": \"...\", \"model_type\": \"llm\"|\"mllm\"}" } ], "return_annotation": "dict", "docstring": "Extract model identity fields from a GET /health response.\n\nArgs:\n data: Parsed JSON body from the /health endpoint. Expected shape::\n\n {\"status\": \"healthy\", \"model_loaded\": True,\n \"model_name\": \"...\", \"model_type\": \"llm\"|\"mllm\"}\n\nReturns:\n ``{\"model_name\": str, \"model_type\": str}``", "summary": "Extract model identity fields from a GET /health response.", "implementation": "Function `parse_health_response` calls `data.get`; returns `{'model_name': data.get('model_name', ''), 'model_type': data.get('model_type', '')}`.", "documented": true, "public": true, "addressable": true, "line": 452, "end_line": 467, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L452-L467", "decorators": [], "calls": [ "data.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'model_name': data.get('model_name', ''), 'model_type': data.get('model_type', '')}" ], "awaits": false, "yields": false }, { "name": "parse_status_response", "qualname": "parse_status_response", "full_name": "vllm_mlx.bench_serve.parse_status_response", "kind": "function", "signature": "def parse_status_response(data: dict) -> dict", "parameters": [ { "name": "data", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Parsed JSON body from the /v1/status endpoint. Metal info is expected under ``data[\"metal\"]`` and cache info under ``data[\"cache\"]``. Missing keys are handled gracefully." } ], "return_annotation": "dict", "docstring": "Extract metal and cache info from a GET /v1/status response.\n\nArgs:\n data: Parsed JSON body from the /v1/status endpoint. Metal info is\n expected under ``data[\"metal\"]`` and cache info under\n ``data[\"cache\"]``. Missing keys are handled gracefully.\n\nReturns:\n ``{\"model\": str, \"metal_active_gb\": float, \"metal_peak_gb\": float,\n \"metal_cache_gb\": float, \"cache_type\": str}``", "summary": "Extract metal and cache info from a GET /v1/status response.", "implementation": "Function `parse_status_response` calls `data.get`, `float`, `metal.get`, `cache.get`; returns `{'model': data.get('model', ''), 'metal_active_gb': float(metal.get('active_memory_gb') or metal.get('active_gb') or 0.…`.", "documented": true, "public": true, "addressable": true, "line": 470, "end_line": 496, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L470-L496", "decorators": [], "calls": [ "data.get", "float", "metal.get", "cache.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'model': data.get('model', ''), 'metal_active_gb': float(metal.get('active_memory_gb') or metal.get('active_gb') or 0.…" ], "awaits": false, "yields": false }, { "name": "parse_metrics_text", "qualname": "parse_metrics_text", "full_name": "vllm_mlx.bench_serve.parse_metrics_text", "kind": "function", "signature": "def parse_metrics_text(text: str) -> dict", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Raw response body from the /metrics endpoint." } ], "return_annotation": "dict", "docstring": "Parse Prometheus text exposition format from GET /metrics.\n\nExtracts the three prefix-cache counters used for bench reporting.\n\nArgs:\n text: Raw response body from the /metrics endpoint.\n\nReturns:\n ``{\"cache_hits\": int, \"cache_misses\": int, \"tokens_saved\": int}``\n — each value defaults to ``0`` when the metric line is absent.", "summary": "Parse Prometheus text exposition format from GET /metrics.", "implementation": "Function `parse_metrics_text` calls `_extract`; returns `{'cache_hits': _extract('vllm_prefix_cache_hits_total'), 'cache_misses': _extract('vllm_prefix_cache_misses_total'), 't…`.", "documented": true, "public": true, "addressable": true, "line": 499, "end_line": 521, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L499-L521", "decorators": [], "calls": [ "_extract" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'cache_hits': _extract('vllm_prefix_cache_hits_total'), 'cache_misses': _extract('vllm_prefix_cache_misses_total'), 't…" ], "awaits": false, "yields": false }, { "name": "_extract", "qualname": "parse_metrics_text._extract", "full_name": "vllm_mlx.bench_serve.parse_metrics_text._extract", "kind": "nested function", "signature": "def _extract(metric_name: str) -> int", "parameters": [ { "name": "metric_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "", "summary": "Nested Function `parse_metrics_text._extract` calls `re.escape`, `re.search`, `int`, `m.group`; returns `int(m.group(1)) if m else 0`.", "implementation": "Nested Function `parse_metrics_text._extract` calls `re.escape`, `re.search`, `int`, `m.group`; returns `int(m.group(1)) if m else 0`.", "documented": false, "public": false, "addressable": false, "line": 512, "end_line": 515, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L512-L515", "decorators": [], "calls": [ "re.escape", "re.search", "int", "m.group" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "int(m.group(1)) if m else 0" ], "awaits": false, "yields": false }, { "name": "detect_hardware_fingerprint", "qualname": "detect_hardware_fingerprint", "full_name": "vllm_mlx.bench_serve.detect_hardware_fingerprint", "kind": "function", "signature": "def detect_hardware_fingerprint() -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Return a hardware fingerprint dict for the current machine.\n\nTries to import :func:`vllm_mlx.optimizations.detect_hardware` (which\nrequires MLX). Falls back to reading ``hw.memsize`` via ``sysctl`` when\nMLX is unavailable. ``os_version`` is always obtained from\n:func:`platform.platform`.\n\nReturns:\n ``{\"chip\": str, \"gpu_cores\": int, \"memory_gb\": float,\n \"bandwidth_gbs\": float, \"os_version\": str}``", "summary": "Return a hardware fingerprint dict for the current machine.", "implementation": "Function `detect_hardware_fingerprint` calls `platform.platform`, `detect_hardware`, `subprocess.run`, `int`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 524, "end_line": 573, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L524-L573", "decorators": [], "calls": [ "platform.platform", "detect_hardware", "subprocess.run", "int", "result.stdout.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'chip': hw.chip_name, 'gpu_cores': hw.gpu_cores, 'memory_gb': hw.total_memory_gb, 'bandwidth_gbs': hw.memory_bandwidth…", "{'chip': '', 'gpu_cores': 0, 'memory_gb': memory_gb, 'bandwidth_gbs': 0.0, 'os_version': os_version}" ], "awaits": false, "yields": false }, { "name": "auto_detect_runtime", "qualname": "auto_detect_runtime", "full_name": "vllm_mlx.bench_serve.auto_detect_runtime", "kind": "function", "signature": "async def auto_detect_runtime(client: httpx.AsyncClient, base_url: str) -> dict", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "An open :class:`httpx.AsyncClient`." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Base URL of the server (e.g. ``\"http://localhost:8080\"``)." } ], "return_annotation": "dict", "docstring": "Query the running server and return a runtime descriptor dict.\n\nHits ``/health``, ``/v1/models``, and ``/v1/status`` in sequence.\nEach call is wrapped in an :exc:`httpx.HTTPError` guard so a missing\nendpoint does not abort the whole detection.\n\nArgs:\n client: An open :class:`httpx.AsyncClient`.\n base_url: Base URL of the server (e.g. ``\"http://localhost:8080\"``).\n\nReturns:\n Dict with keys: ``model_id``, ``model_type``, ``engine_type``,\n ``mtp_enabled``, ``specprefill``, ``kv_quant``, ``cache_type``,\n ``metal_active_gb``, ``metal_peak_gb``, ``metal_cache_gb``.", "summary": "Query the running server and return a runtime descriptor dict.", "implementation": "Function `auto_detect_runtime` calls `client.get`, `resp.raise_for_status`, `parse_health_response`, `resp.json`; awaits asynchronous work; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 576, "end_line": 642, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L576-L642", "decorators": [], "calls": [ "client.get", "resp.raise_for_status", "parse_health_response", "resp.json", "health.get", "models_data.get", "models[0].get", "parse_status_response", "status.get", "raw.get", "bool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": true, "yields": false }, { "name": "scrape_metrics", "qualname": "scrape_metrics", "full_name": "vllm_mlx.bench_serve.scrape_metrics", "kind": "function", "signature": "async def scrape_metrics(client: httpx.AsyncClient, base_url: str) -> dict", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "An open :class:`httpx.AsyncClient`." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Base URL of the server." } ], "return_annotation": "dict", "docstring": "Scrape Prometheus metrics from the server.\n\nArgs:\n client: An open :class:`httpx.AsyncClient`.\n base_url: Base URL of the server.\n\nReturns:\n Parsed metrics dict (see :func:`parse_metrics_text`), or an empty\n dict if the endpoint is unreachable.", "summary": "Scrape Prometheus metrics from the server.", "implementation": "Function `scrape_metrics` calls `client.get`, `resp.raise_for_status`, `parse_metrics_text`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 645, "end_line": 661, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L645-L661", "decorators": [], "calls": [ "client.get", "resp.raise_for_status", "parse_metrics_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parse_metrics_text(resp.text)", "{}" ], "awaits": true, "yields": false }, { "name": "clear_runtime_cache", "qualname": "clear_runtime_cache", "full_name": "vllm_mlx.bench_serve.clear_runtime_cache", "kind": "function", "signature": "async def clear_runtime_cache(client: httpx.AsyncClient, base_url: str) -> dict", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Clear server-side runtime caches and return a JSON-serializable event.", "summary": "Clear server-side runtime caches and return a JSON-serializable event.", "implementation": "Function `clear_runtime_cache` calls `client.delete`, `resp.json`, `resp.raise_for_status`, `str`; awaits asynchronous work; returns `event`.", "documented": true, "public": true, "addressable": true, "line": 664, "end_line": 684, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L664-L684", "decorators": [], "calls": [ "client.delete", "resp.json", "resp.raise_for_status", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "event" ], "awaits": true, "yields": false }, { "name": "_normalize_cache_policy", "qualname": "_normalize_cache_policy", "full_name": "vllm_mlx.bench_serve._normalize_cache_policy", "kind": "function", "signature": "def _normalize_cache_policy(value: Optional[str]) -> str", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Normalize cache-policy spelling from CLI or workload JSON.\n\nCLI choices are hyphenated, but workload JSON may use underscores when it\nfollows common Python/YAML identifier style.", "summary": "Normalize cache-policy spelling from CLI or workload JSON.", "implementation": "Function `_normalize_cache_policy` calls `(value or 'preserve').strip().lower().replace`, `(value or 'preserve').strip().lower`, `(value or 'preserve').strip`, `ValueError`; can raise `ValueError`; returns `policy`.", "documented": true, "public": false, "addressable": true, "line": 687, "end_line": 698, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L687-L698", "decorators": [], "calls": [ "(value or 'preserve').strip().lower().replace", "(value or 'preserve').strip().lower", "(value or 'preserve').strip", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "policy" ], "awaits": false, "yields": false }, { "name": "parse_sse_line", "qualname": "parse_sse_line", "full_name": "vllm_mlx.bench_serve.parse_sse_line", "kind": "function", "signature": "def parse_sse_line(line: str) -> Optional[dict]", "parameters": [ { "name": "line", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "A single raw line from the SSE stream (may or may not include a trailing newline — it is stripped before processing)." } ], "return_annotation": "Optional[dict]", "docstring": "Parse one Server-Sent Events line from a streaming chat completion.\n\nArgs:\n line: A single raw line from the SSE stream (may or may not include\n a trailing newline — it is stripped before processing).\n\nReturns:\n ``None`` for blank lines, comment lines (starting with ``:``) and the\n ``data: [DONE]`` sentinel. For all other ``data:`` lines the JSON is\n parsed and a dict is returned::\n\n {\"id\": Optional[str], \"content\": str,\n \"finish_reason\": Optional[str], \"usage\": Optional[dict],\n \"tool_calls_delta\": Optional[list]}\n\n Missing keys (``choices``, ``delta``, ``content``) are handled\n gracefully and default to empty string / ``None``.", "summary": "Parse one Server-Sent Events line from a streaming chat completion.", "implementation": "Function `parse_sse_line` calls `line.strip`, `line.startswith`, `len`, `json.loads`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 706, "end_line": 754, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L706-L754", "decorators": [], "calls": [ "line.strip", "line.startswith", "len", "json.loads", "chunk.get", "choices[0].get", "delta.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "{'id': chunk.get('id'), 'content': content, 'finish_reason': finish_reason, 'usage': usage, 'tool_calls_delta': tool_ca…" ], "awaits": false, "yields": false }, { "name": "_cancel_server_request", "qualname": "_cancel_server_request", "full_name": "vllm_mlx.bench_serve._cancel_server_request", "kind": "function", "signature": "async def _cancel_server_request(client: httpx.AsyncClient, base_url: str, request_id: Optional[str]) -> None", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request_id", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Best-effort server-side cancellation for timed-out workload streams.", "summary": "Best-effort server-side cancellation for timed-out workload streams.", "implementation": "Function `_cancel_server_request` calls `client.post`; awaits asynchronous work; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 757, "end_line": 770, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L757-L770", "decorators": [], "calls": [ "client.post" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "accumulate_tool_calls", "qualname": "accumulate_tool_calls", "full_name": "vllm_mlx.bench_serve.accumulate_tool_calls", "kind": "function", "signature": "def accumulate_tool_calls(acc: dict[int, dict], delta_list: list[dict]) -> None", "parameters": [ { "name": "acc", "kind": "positional or keyword", "annotation": "dict[int, dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_list", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Merge streamed OpenAI tool-call deltas into *acc* by index.", "summary": "Merge streamed OpenAI tool-call deltas into *acc* by index.", "implementation": "Function `accumulate_tool_calls` calls `int`, `tc_delta.get`, `function_delta.get`.", "documented": true, "public": true, "addressable": true, "line": 773, "end_line": 792, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L773-L792", "decorators": [], "calls": [ "int", "tc_delta.get", "function_delta.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "finalize_tool_calls", "qualname": "finalize_tool_calls", "full_name": "vllm_mlx.bench_serve.finalize_tool_calls", "kind": "function", "signature": "def finalize_tool_calls(acc: dict[int, dict]) -> list[dict]", "parameters": [ { "name": "acc", "kind": "positional or keyword", "annotation": "dict[int, dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Return accumulated tool calls in stream index order.", "summary": "Return accumulated tool calls in stream index order.", "implementation": "Function `finalize_tool_calls` calls `sorted`; returns `[acc[idx] for idx in sorted(acc)]`.", "documented": true, "public": true, "addressable": true, "line": 795, "end_line": 797, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L795-L797", "decorators": [], "calls": [ "sorted" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[acc[idx] for idx in sorted(acc)]" ], "awaits": false, "yields": false }, { "name": "compute_request_metrics", "qualname": "compute_request_metrics", "full_name": "vllm_mlx.bench_serve.compute_request_metrics", "kind": "function", "signature": "def compute_request_metrics(t_start: float, t_first_token: float, token_times: list, t_end: float, prompt_tokens: int, completion_tokens: int) -> dict", "parameters": [ { "name": "t_start", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Timestamp immediately before the request was sent." }, { "name": "t_first_token", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Timestamp when the first content token was received." }, { "name": "token_times", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "List of timestamps, one per content token (including the first). When there is only one token ``tpot_ms`` is ``0.0``." }, { "name": "t_end", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Timestamp after the final SSE chunk was consumed." }, { "name": "prompt_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of prompt tokens reported by the server." }, { "name": "completion_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of completion tokens generated." } ], "return_annotation": "dict", "docstring": "Compute standard latency and throughput metrics for a single request.\n\nAll time arguments are :func:`time.perf_counter` values (seconds as\nfloats).\n\nArgs:\n t_start: Timestamp immediately before the request was sent.\n t_first_token: Timestamp when the first content token was received.\n token_times: List of timestamps, one per content token (including the\n first). When there is only one token ``tpot_ms`` is ``0.0``.\n t_end: Timestamp after the final SSE chunk was consumed.\n prompt_tokens: Number of prompt tokens reported by the server.\n completion_tokens: Number of completion tokens generated.\n\nReturns:\n Dict with keys ``ttft_ms``, ``tpot_ms``, ``e2e_latency_ms``,\n ``gen_tps``, ``prompt_tps`` — all floats.", "summary": "Compute standard latency and throughput metrics for a single request.", "implementation": "Function `compute_request_metrics` calls `len`, `range`, `statistics.mean`; returns `{'ttft_ms': ttft_ms, 'tpot_ms': tpot_ms, 'e2e_latency_ms': e2e_latency_ms, 'gen_tps': gen_tps, 'prompt_tps': prompt_tps}`.", "documented": true, "public": true, "addressable": true, "line": 800, "end_line": 852, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L800-L852", "decorators": [], "calls": [ "len", "range", "statistics.mean" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'ttft_ms': ttft_ms, 'tpot_ms': tpot_ms, 'e2e_latency_ms': e2e_latency_ms, 'gen_tps': gen_tps, 'prompt_tps': prompt_tps}" ], "awaits": false, "yields": false }, { "name": "count_prompt_tokens", "qualname": "count_prompt_tokens", "full_name": "vllm_mlx.bench_serve.count_prompt_tokens", "kind": "function", "signature": "async def count_prompt_tokens(client: httpx.AsyncClient, base_url: str, messages: list[dict], model: str) -> int", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "An open :class:`httpx.AsyncClient`." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Base URL of the server." }, { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "The message list to send." }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Model ID to target." } ], "return_annotation": "int", "docstring": "Count prompt tokens for a message list by sending a 1-token request.\n\nSends a non-streaming chat completion with ``max_tokens=1`` and reads\n``usage.prompt_tokens`` from the response.\n\nArgs:\n client: An open :class:`httpx.AsyncClient`.\n base_url: Base URL of the server.\n messages: The message list to send.\n model: Model ID to target.\n\nReturns:\n Number of prompt tokens, or ``0`` on error.", "summary": "Count prompt tokens for a message list by sending a 1-token request.", "implementation": "Function `count_prompt_tokens` calls `client.post`, `resp.raise_for_status`, `resp.json`, `int`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 855, "end_line": 889, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L855-L889", "decorators": [], "calls": [ "client.post", "resp.raise_for_status", "resp.json", "int", "(data.get('usage') or {}).get", "data.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "int((data.get('usage') or {}).get('prompt_tokens', 0))", "0" ], "awaits": true, "yields": false }, { "name": "stream_chat_completion", "qualname": "stream_chat_completion", "full_name": "vllm_mlx.bench_serve.stream_chat_completion", "kind": "function", "signature": "async def stream_chat_completion(client: httpx.AsyncClient, base_url: str, messages: list[dict], model: str, max_tokens: int=256, enable_thinking: Optional[bool]=None, extra_body: Optional[dict]=None, timeout_s: Optional[float]=None) -> dict", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "An open :class:`httpx.AsyncClient`." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Base URL of the server." }, { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "The message list to send." }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Model ID to target." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate (default ``256``)." }, { "name": "enable_thinking", "kind": "positional or keyword", "annotation": "Optional[bool]", "default": "None", "required": false, "description": "If not ``None``, passed as ``enable_thinking`` in the request body." }, { "name": "extra_body", "kind": "positional or keyword", "annotation": "Optional[dict]", "default": "None", "required": false, "description": "Optional extra keys merged into the request body." }, { "name": "timeout_s", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional case-level timeout. When set, the stream is closed and best-effort server cancellation is attempted before raising :class:`TimeoutError`." } ], "return_annotation": "dict", "docstring": "Send a streaming chat completion and collect per-token timing data.\n\nTracks TTFT, per-token timestamps, accumulated content, finish reason,\nand usage (via ``stream_options: {\"include_usage\": True}``).\n\nArgs:\n client: An open :class:`httpx.AsyncClient`.\n base_url: Base URL of the server.\n messages: The message list to send.\n model: Model ID to target.\n max_tokens: Maximum tokens to generate (default ``256``).\n enable_thinking: If not ``None``, passed as ``enable_thinking`` in the\n request body.\n extra_body: Optional extra keys merged into the request body.\n timeout_s: Optional case-level timeout. When set, the stream is closed\n and best-effort server cancellation is attempted before raising\n :class:`TimeoutError`.\n\nReturns:\n Dict with all :func:`compute_request_metrics` fields plus\n ``completion_tokens``, ``prompt_tokens``, ``finish_reason``,\n ``content``.", "summary": "Send a streaming chat completion and collect per-token timing data.", "implementation": "Function `stream_chat_completion` calls `body.update`, `time.perf_counter`, `asyncio.timeout`, `_consume_stream`; awaits asynchronous work; can raise `TimeoutError`; returns `{**metrics, 'completion_tokens': completion_tokens, 'prompt_tokens': prompt_tokens, 'finish_reason': finish_reason, 'co…`.", "documented": true, "public": true, "addressable": true, "line": 892, "end_line": 1012, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L892-L1012", "decorators": [], "calls": [ "body.update", "time.perf_counter", "asyncio.timeout", "_consume_stream", "_cancel_server_request", "TimeoutError", "int", "(usage or {}).get", "compute_request_metrics", "''.join", "finalize_tool_calls" ], "state_reads": [], "state_writes": [], "raises": [ "TimeoutError" ], "return_expressions": [ "{**metrics, 'completion_tokens': completion_tokens, 'prompt_tokens': prompt_tokens, 'finish_reason': finish_reason, 'co…" ], "awaits": true, "yields": false }, { "name": "_consume_stream", "qualname": "stream_chat_completion._consume_stream", "full_name": "vllm_mlx.bench_serve.stream_chat_completion._consume_stream", "kind": "nested function", "signature": "async def _consume_stream() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `stream_chat_completion._consume_stream` calls `client.stream`, `response.raise_for_status`, `response.aiter_lines`, `parse_sse_line`.", "implementation": "Nested Function `stream_chat_completion._consume_stream` calls `client.stream`, `response.raise_for_status`, `response.aiter_lines`, `parse_sse_line`.", "documented": false, "public": false, "addressable": false, "line": 946, "end_line": 975, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L946-L975", "decorators": [], "calls": [ "client.stream", "response.raise_for_status", "response.aiter_lines", "parse_sse_line", "parsed.get", "time.perf_counter", "token_times.append", "accumulate_tool_calls", "content_parts.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "validate_response", "qualname": "validate_response", "full_name": "vllm_mlx.bench_serve.validate_response", "kind": "function", "signature": "def validate_response(finish_reason: Optional[str], content: str, status_code: int, *, tool_calls: Optional[list[dict]]=None) -> tuple[bool, str]", "parameters": [ { "name": "finish_reason", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "", "required": true, "description": "The ``finish_reason`` from the final SSE chunk, or ``None`` if not received." }, { "name": "content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The accumulated text content of the response." }, { "name": "status_code", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "The HTTP status code of the response (use ``200`` for successful streaming requests)." }, { "name": "tool_calls", "kind": "keyword-only", "annotation": "Optional[list[dict]]", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "tuple[bool, str]", "docstring": "Validate a single streaming response result.\n\nArgs:\n finish_reason: The ``finish_reason`` from the final SSE chunk, or\n ``None`` if not received.\n content: The accumulated text content of the response.\n status_code: The HTTP status code of the response (use ``200`` for\n successful streaming requests).\n\nReturns:\n ``(is_valid, message)`` — ``is_valid`` is ``True`` when the response\n passes all checks; ``message`` is an empty string on success or a\n human-readable description of the first failure.", "summary": "Validate a single streaming response result.", "implementation": "Function `validate_response` has 5 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1020, "end_line": 1049, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1020-L1049", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(False, f'HTTP error {status_code}')", "(False, 'Missing finish_reason')", "(False, 'Truncated (finish_reason=length)')", "(False, 'Empty response content')", "(True, '')" ], "awaits": false, "yields": false }, { "name": "_check_finish_reason", "qualname": "_check_finish_reason", "full_name": "vllm_mlx.bench_serve._check_finish_reason", "kind": "function", "signature": "def _check_finish_reason(allowed: Any, finish_reason: Optional[str]) -> list[str]", "parameters": [ { "name": "allowed", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "finish_reason", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Verify ``finish_reason`` is in the allowed set, if one is configured.", "summary": "Verify ``finish_reason`` is in the allowed set, if one is configured.", "implementation": "Function `_check_finish_reason` calls `isinstance`, `list`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1052, "end_line": 1059, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1052-L1059", "decorators": [], "calls": [ "isinstance", "list" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "[f'finish_reason {finish_reason!r} not in allowed set {allowed_list!r}']" ], "awaits": false, "yields": false }, { "name": "_check_length_bounds", "qualname": "_check_length_bounds", "full_name": "vllm_mlx.bench_serve._check_length_bounds", "kind": "function", "signature": "def _check_length_bounds(min_chars: Any, max_chars: Any, content: str) -> list[str]", "parameters": [ { "name": "min_chars", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_chars", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Apply ``min_chars`` / ``max_chars`` content-length bounds.", "summary": "Apply ``min_chars`` / ``max_chars`` content-length bounds.", "implementation": "Function `_check_length_bounds` calls `len`, `int`, `issues.append`; returns `issues`.", "documented": true, "public": false, "addressable": true, "line": 1062, "end_line": 1069, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1062-L1069", "decorators": [], "calls": [ "len", "int", "issues.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "issues" ], "awaits": false, "yields": false }, { "name": "_check_regex_patterns", "qualname": "_check_regex_patterns", "full_name": "vllm_mlx.bench_serve._check_regex_patterns", "kind": "function", "signature": "def _check_regex_patterns(patterns: Any, content: str, *, kind: str, expect_match: bool) -> list[str]", "parameters": [ { "name": "patterns", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "kind", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "expect_match", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "list[str]", "docstring": "Validate that each pattern either matches or does not, per ``expect_match``.\n\n``kind`` is the diagnostic name (``\"required_regex\"`` or\n``\"forbidden_regex\"``) and is reused across the resulting issue\nstrings so operators can grep for the failing check.", "summary": "Validate that each pattern either matches or does not, per ``expect_match``.", "implementation": "Function `_check_regex_patterns` calls `bool`, `re.search`, `str`, `issues.append`; returns `issues`.", "documented": true, "public": false, "addressable": true, "line": 1072, "end_line": 1096, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1072-L1096", "decorators": [], "calls": [ "bool", "re.search", "str", "issues.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "issues" ], "awaits": false, "yields": false }, { "name": "_check_json_content", "qualname": "_check_json_content", "full_name": "vllm_mlx.bench_serve._check_json_content", "kind": "function", "signature": "def _check_json_content(should_be_json: Any, content: str) -> list[str]", "parameters": [ { "name": "should_be_json", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Verify ``content`` parses as JSON when ``checks['json']`` is truthy.", "summary": "Verify ``content`` parses as JSON when ``checks['json']`` is truthy.", "implementation": "Function `_check_json_content` calls `json.loads`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1099, "end_line": 1107, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1099-L1107", "decorators": [], "calls": [ "json.loads" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "[f'content is not valid JSON: {exc}']" ], "awaits": false, "yields": false }, { "name": "_check_tool_call_count_and_names", "qualname": "_check_tool_call_count_and_names", "full_name": "vllm_mlx.bench_serve._check_tool_call_count_and_names", "kind": "function", "signature": "def _check_tool_call_count_and_names(checks: dict, tool_calls: list[dict]) -> list[str]", "parameters": [ { "name": "checks", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tool_calls", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Apply ``no_tool_calls`` / ``tool_call_count`` / ``tool_call_names``.", "summary": "Apply ``no_tool_calls`` / ``tool_call_count`` / ``tool_call_names``.", "implementation": "Function `_check_tool_call_count_and_names` calls `checks.get`, `issues.append`, `len`, `int`; returns `issues`.", "documented": true, "public": false, "addressable": true, "line": 1110, "end_line": 1132, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1110-L1132", "decorators": [], "calls": [ "checks.get", "issues.append", "len", "int", "sorted", "tc.get('function', {}).get", "tc.get", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "issues" ], "awaits": false, "yields": false }, { "name": "_check_tool_call_args", "qualname": "_check_tool_call_args", "full_name": "vllm_mlx.bench_serve._check_tool_call_args", "kind": "function", "signature": "def _check_tool_call_args(required_args: Any, tool_calls: list[dict]) -> list[str]", "parameters": [ { "name": "required_args", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tool_calls", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Validate parsed JSON arguments include the required keys per function.\n\nFor each named function, looks up matching tool calls, parses their\n``arguments`` as JSON, and reports issues for: missing tool call,\ninvalid JSON, non-object arguments, or missing required keys.", "summary": "Validate parsed JSON arguments include the required keys per function.", "implementation": "Function `_check_tool_call_args` calls `tc.get('function', {}).get`, `tc.get`, `by_name.setdefault(name, []).append`, `by_name.setdefault`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1135, "end_line": 1174, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1135-L1174", "decorators": [], "calls": [ "tc.get('function', {}).get", "tc.get", "by_name.setdefault(name, []).append", "by_name.setdefault", "required_args.items", "by_name.get", "str", "issues.append", "json.loads", "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "issues" ], "awaits": false, "yields": false }, { "name": "validate_quality_checks", "qualname": "validate_quality_checks", "full_name": "vllm_mlx.bench_serve.validate_quality_checks", "kind": "function", "signature": "def validate_quality_checks(finish_reason: Optional[str], content: str, checks: Optional[dict], *, status_code: int=200, tool_calls: Optional[list[dict]]=None) -> tuple[bool, list[str]]", "parameters": [ { "name": "finish_reason", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "checks", "kind": "positional or keyword", "annotation": "Optional[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "status_code", "kind": "keyword-only", "annotation": "int", "default": "200", "required": false, "description": "Optional keyword-only input; defaults to `200`." }, { "name": "tool_calls", "kind": "keyword-only", "annotation": "Optional[list[dict]]", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "tuple[bool, list[str]]", "docstring": "Validate content against generic workload quality checks.\n\nSupported checks:\n- ``finish_reason``: string or list of allowed finish reasons\n- ``required_regex``: list of regex patterns that must match\n- ``forbidden_regex``: list of regex patterns that must not match\n- ``min_chars`` / ``max_chars``: length bounds\n- ``json``: when true, content must parse as JSON\n- ``tool_call_count``: exact number of streamed tool calls\n- ``tool_call_names``: expected function names, order-independent\n- ``tool_call_args_required_keys``: required JSON argument keys by function\n- ``no_tool_calls``: assert that no tool calls were emitted", "summary": "Validate content against generic workload quality checks.", "implementation": "Function `validate_quality_checks` calls `validate_response`, `issues.extend`, `_check_finish_reason`, `checks.get`; returns `(not issues, issues)`.", "documented": true, "public": true, "addressable": true, "line": 1177, "end_line": 1231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1177-L1231", "decorators": [], "calls": [ "validate_response", "issues.extend", "_check_finish_reason", "checks.get", "_check_length_bounds", "_check_regex_patterns", "_check_json_content", "_check_tool_call_count_and_names", "_check_tool_call_args" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(not issues, issues)" ], "awaits": false, "yields": false }, { "name": "compute_summary_stats", "qualname": "compute_summary_stats", "full_name": "vllm_mlx.bench_serve.compute_summary_stats", "kind": "function", "signature": "def compute_summary_stats(values: list[float]) -> dict", "parameters": [ { "name": "values", "kind": "positional or keyword", "annotation": "list[float]", "default": "", "required": true, "description": "Non-empty list of floats to summarise." } ], "return_annotation": "dict", "docstring": "Compute summary statistics over a list of floats.\n\nArgs:\n values: Non-empty list of floats to summarise.\n\nReturns:\n Dict with keys ``mean``, ``stddev``, ``min``, ``max``, ``p50``,\n ``p95``, ``p99``. Percentiles use linear interpolation on sorted\n values.\n\nRaises:\n ValueError: If ``values`` is empty.", "summary": "Compute summary statistics over a list of floats.", "implementation": "Function `compute_summary_stats` calls `ValueError`, `len`, `statistics.mean`, `statistics.stdev`; can raise `ValueError`; returns `{'mean': mean, 'stddev': stddev, 'min': sorted_vals[0], 'max': sorted_vals[-1], 'p50': _percentile(50), 'p95': _percent…`.", "documented": true, "public": true, "addressable": true, "line": 1234, "end_line": 1276, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1234-L1276", "decorators": [], "calls": [ "ValueError", "len", "statistics.mean", "statistics.stdev", "sorted", "_percentile" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "{'mean': mean, 'stddev': stddev, 'min': sorted_vals[0], 'max': sorted_vals[-1], 'p50': _percentile(50), 'p95': _percent…" ], "awaits": false, "yields": false }, { "name": "_percentile", "qualname": "compute_summary_stats._percentile", "full_name": "vllm_mlx.bench_serve.compute_summary_stats._percentile", "kind": "nested function", "signature": "def _percentile(p: float) -> float", "parameters": [ { "name": "p", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "", "summary": "Nested Function `compute_summary_stats._percentile` calls `int`; has 3 explicit return paths.", "implementation": "Nested Function `compute_summary_stats._percentile` calls `int`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 1256, "end_line": 1266, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1256-L1266", "decorators": [], "calls": [ "int" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "sorted_vals[0]", "sorted_vals[-1]", "sorted_vals[lo] + frac * (sorted_vals[hi] - sorted_vals[lo])" ], "awaits": false, "yields": false }, { "name": "run_concurrent_requests", "qualname": "run_concurrent_requests", "full_name": "vllm_mlx.bench_serve.run_concurrent_requests", "kind": "function", "signature": "async def run_concurrent_requests(client: httpx.AsyncClient, base_url: str, prompts: list[list[dict]], model: str, concurrency: int, max_tokens: int=256, enable_thinking: Optional[bool]=None, extra_body: Optional[dict]=None, do_validate: bool=True) -> list[dict]", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "An open :class:`httpx.AsyncClient`." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Base URL of the server." }, { "name": "prompts", "kind": "positional or keyword", "annotation": "list[list[dict]]", "default": "", "required": true, "description": "List of message dicts to cycle through." }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Model ID to target." }, { "name": "concurrency", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of simultaneous requests to fire." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate per request (default ``256``)." }, { "name": "enable_thinking", "kind": "positional or keyword", "annotation": "Optional[bool]", "default": "None", "required": false, "description": "Passed through to :func:`stream_chat_completion`." }, { "name": "extra_body", "kind": "positional or keyword", "annotation": "Optional[dict]", "default": "None", "required": false, "description": "Passed through to :func:`stream_chat_completion`." }, { "name": "do_validate", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "When ``True``, call :func:`validate_response` on each result and add a ``\"validated\"`` key." } ], "return_annotation": "list[dict]", "docstring": "Fire ``concurrency`` concurrent streaming requests and collect results.\n\nPrompts are selected round-robin from ``prompts``. All requests are\nlaunched simultaneously with :func:`asyncio.gather`. Exceptions are\ncaught per-task and wrapped in an error dict rather than propagated.\n\nArgs:\n client: An open :class:`httpx.AsyncClient`.\n base_url: Base URL of the server.\n prompts: List of message dicts to cycle through.\n model: Model ID to target.\n concurrency: Number of simultaneous requests to fire.\n max_tokens: Maximum tokens to generate per request (default ``256``).\n enable_thinking: Passed through to :func:`stream_chat_completion`.\n extra_body: Passed through to :func:`stream_chat_completion`.\n do_validate: When ``True``, call :func:`validate_response` on each\n result and add a ``\"validated\"`` key.\n\nReturns:\n List of result dicts (one per request). Each dict contains at minimum\n a ``\"validated\"`` key when ``do_validate`` is ``True``.", "summary": "Fire ``concurrency`` concurrent streaming requests and collect results.", "implementation": "Function `run_concurrent_requests` calls `itertools.cycle`, `next`, `range`, `asyncio.gather`; awaits asynchronous work; returns `list(results)`.", "documented": true, "public": true, "addressable": true, "line": 1279, "end_line": 1342, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1279-L1342", "decorators": [], "calls": [ "itertools.cycle", "next", "range", "asyncio.gather", "_single", "list" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "list(results)" ], "awaits": true, "yields": false }, { "name": "_single", "qualname": "run_concurrent_requests._single", "full_name": "vllm_mlx.bench_serve.run_concurrent_requests._single", "kind": "nested function", "signature": "async def _single(messages: list[dict]) -> dict", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "", "summary": "Nested Function `run_concurrent_requests._single` calls `stream_chat_completion`, `validate_response`, `result.get`, `str`; awaits asynchronous work; has 2 explicit return paths.", "implementation": "Nested Function `run_concurrent_requests._single` calls `stream_chat_completion`, `validate_response`, `result.get`, `str`; awaits asynchronous work; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 1315, "end_line": 1339, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1315-L1339", "decorators": [], "calls": [ "stream_chat_completion", "validate_response", "result.get", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result", "err" ], "awaits": true, "yields": false }, { "name": "_summary_or_empty", "qualname": "_summary_or_empty", "full_name": "vllm_mlx.bench_serve._summary_or_empty", "kind": "function", "signature": "def _summary_or_empty(values: list[float]) -> dict", "parameters": [ { "name": "values", "kind": "positional or keyword", "annotation": "list[float]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "", "summary": "Function `_summary_or_empty` calls `compute_summary_stats`; returns `compute_summary_stats(values) if values else {}`.", "implementation": "Function `_summary_or_empty` calls `compute_summary_stats`; returns `compute_summary_stats(values) if values else {}`.", "documented": false, "public": false, "addressable": true, "line": 1345, "end_line": 1346, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1345-L1346", "decorators": [], "calls": [ "compute_summary_stats" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "compute_summary_stats(values) if values else {}" ], "awaits": false, "yields": false }, { "name": "_resolve_max_tokens", "qualname": "_resolve_max_tokens", "full_name": "vllm_mlx.bench_serve._resolve_max_tokens", "kind": "function", "signature": "def _resolve_max_tokens(case: WorkloadCase, workload: Workload) -> int", "parameters": [ { "name": "case", "kind": "positional or keyword", "annotation": "WorkloadCase", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "workload", "kind": "positional or keyword", "annotation": "Workload", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Return the effective ``max_tokens`` for a case, falling back to\nworkload defaults and finally to 256.", "summary": "Return the effective ``max_tokens`` for a case, falling back to workload defaults and finally to 256.", "implementation": "Function `_resolve_max_tokens` calls `int`, `workload.defaults.get`; returns `int(case.max_tokens or workload.defaults.get('max_tokens', 256))`.", "documented": true, "public": false, "addressable": true, "line": 1349, "end_line": 1352, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1349-L1352", "decorators": [], "calls": [ "int", "workload.defaults.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "int(case.max_tokens or workload.defaults.get('max_tokens', 256))" ], "awaits": false, "yields": false }, { "name": "_assemble_case_request_kwargs", "qualname": "_assemble_case_request_kwargs", "full_name": "vllm_mlx.bench_serve._assemble_case_request_kwargs", "kind": "function", "signature": "def _assemble_case_request_kwargs(case: WorkloadCase, workload: Workload, model: str) -> dict", "parameters": [ { "name": "case", "kind": "positional or keyword", "annotation": "WorkloadCase", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "workload", "kind": "positional or keyword", "annotation": "Workload", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Build the keyword-arguments dict passed to ``stream_chat_completion``\nfor one case, applying max_tokens fallback and converting\n``policy_timeout_ms`` to seconds.", "summary": "Build the keyword-arguments dict passed to ``stream_chat_completion`` for one case, applying max_tokens fallback and converting ``policy_timeout_ms`` to seconds.", "implementation": "Function `_assemble_case_request_kwargs` calls `_resolve_max_tokens`; returns `{'messages': case.messages, 'model': model, 'max_tokens': _resolve_max_tokens(case, workload), 'enable_thinking': case.…`.", "documented": true, "public": false, "addressable": true, "line": 1355, "end_line": 1372, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1355-L1372", "decorators": [], "calls": [ "_resolve_max_tokens" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'messages': case.messages, 'model': model, 'max_tokens': _resolve_max_tokens(case, workload), 'enable_thinking': case.…" ], "awaits": false, "yields": false }, { "name": "_empty_completion_result", "qualname": "_empty_completion_result", "full_name": "vllm_mlx.bench_serve._empty_completion_result", "kind": "function", "signature": "def _empty_completion_result() -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Zero-valued completion result used when ``stream_chat_completion``\nraises. The structure matches a real successful response so downstream\ncode can read ``result.get(...)`` without branching on the failure.", "summary": "Zero-valued completion result used when ``stream_chat_completion`` raises.", "implementation": "Function `_empty_completion_result` returns `{'ttft_ms': 0.0, 'tpot_ms': 0.0, 'e2e_latency_ms': 0.0, 'gen_tps': 0.0, 'prompt_tps': 0.0, 'prompt_tokens': 0, 'complet…`.", "documented": true, "public": false, "addressable": true, "line": 1375, "end_line": 1390, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1375-L1390", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'ttft_ms': 0.0, 'tpot_ms': 0.0, 'e2e_latency_ms': 0.0, 'gen_tps': 0.0, 'prompt_tps': 0.0, 'prompt_tokens': 0, 'complet…" ], "awaits": false, "yields": false }, { "name": "_fetch_post_run_status", "qualname": "_fetch_post_run_status", "full_name": "vllm_mlx.bench_serve._fetch_post_run_status", "kind": "function", "signature": "async def _fetch_post_run_status(client: httpx.AsyncClient, base_url: str) -> dict", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "GET ``/v1/status`` after a case run, swallowing transport errors so\na missing or temporarily-unavailable status endpoint does not fail\nthe case record.", "summary": "GET ``/v1/status`` after a case run, swallowing transport errors so a missing or temporarily-unavailable status endpoint does not fail the case record.", "implementation": "Function `_fetch_post_run_status` calls `client.get`, `resp.raise_for_status`, `resp.json`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1393, "end_line": 1402, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1393-L1402", "decorators": [], "calls": [ "client.get", "resp.raise_for_status", "resp.json" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "resp.json()", "{}" ], "awaits": true, "yields": false }, { "name": "_compute_within_policy_timeout", "qualname": "_compute_within_policy_timeout", "full_name": "vllm_mlx.bench_serve._compute_within_policy_timeout", "kind": "function", "signature": "def _compute_within_policy_timeout(timeout_ms: Optional[int], *, error_present: bool, e2e_latency_ms: float) -> Optional[bool]", "parameters": [ { "name": "timeout_ms", "kind": "positional or keyword", "annotation": "Optional[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "error_present", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "e2e_latency_ms", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "Optional[bool]", "docstring": "Resolve the ``policy.within_timeout`` field.\n\n``None`` when the case did not configure a policy timeout, ``False``\nwhen the request errored (any latency claim would be misleading),\nand otherwise the latency comparison result.", "summary": "Resolve the ``policy.within_timeout`` field.", "implementation": "Function `_compute_within_policy_timeout` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1405, "end_line": 1418, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1405-L1418", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "False", "e2e_latency_ms <= timeout_ms" ], "awaits": false, "yields": false }, { "name": "_build_tool_calls_summary", "qualname": "_build_tool_calls_summary", "full_name": "vllm_mlx.bench_serve._build_tool_calls_summary", "kind": "function", "signature": "def _build_tool_calls_summary(tool_calls: Any) -> Optional[dict]", "parameters": [ { "name": "tool_calls", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[dict]", "docstring": "Compact summary of streamed tool calls for the case record.\n\nReturns ``None`` when no tool calls were emitted so consumers can\ndistinguish \"feature not exercised\" from \"feature exercised, zero\ncalls\" if that ever matters.", "summary": "Compact summary of streamed tool calls for the case record.", "implementation": "Function `_build_tool_calls_summary` calls `len`, `sorted`, `tc.get('function', {}).get`, `tc.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1421, "end_line": 1434, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1421-L1434", "decorators": [], "calls": [ "len", "sorted", "tc.get('function', {}).get", "tc.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "{'count': len(tool_calls), 'names': sorted((tc.get('function', {}).get('name', '') for tc in tool_calls)), 'raw': tool_…" ], "awaits": false, "yields": false }, { "name": "_build_workload_record", "qualname": "_build_workload_record", "full_name": "vllm_mlx.bench_serve._build_workload_record", "kind": "function", "signature": "def _build_workload_record(*, case: WorkloadCase, workload: Workload, model: str, runtime: dict, hardware: dict, run_id: str, timestamp: str, started_wall: str, repetition: int, result: dict, error: str, quality_ok: bool, quality_issues: list[str], content: str, cache_hits_delta: int, cache_misses_delta: int, tokens_saved_delta: int, status_after: dict, cache_reset: Optional[dict], include_content: bool) -> dict", "parameters": [ { "name": "case", "kind": "keyword-only", "annotation": "WorkloadCase", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "workload", "kind": "keyword-only", "annotation": "Workload", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "model", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "runtime", "kind": "keyword-only", "annotation": "dict", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "hardware", "kind": "keyword-only", "annotation": "dict", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "run_id", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "timestamp", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "started_wall", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "repetition", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "result", "kind": "keyword-only", "annotation": "dict", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "error", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "quality_ok", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "quality_issues", "kind": "keyword-only", "annotation": "list[str]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "content", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "cache_hits_delta", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "cache_misses_delta", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "tokens_saved_delta", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "status_after", "kind": "keyword-only", "annotation": "dict", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "cache_reset", "kind": "keyword-only", "annotation": "Optional[dict]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "include_content", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "dict", "docstring": "Assemble the JSON-serializable workload-case record from the raw\ninputs and the completion result. Pure function: no I/O, deterministic\ngiven its arguments.", "summary": "Assemble the JSON-serializable workload-case record from the raw inputs and the completion result.", "implementation": "Function `_build_workload_record` calls `list`, `_resolve_max_tokens`, `len`, `_compute_within_policy_timeout`; returns `record`.", "documented": true, "public": false, "addressable": true, "line": 1437, "end_line": 1515, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1437-L1515", "decorators": [], "calls": [ "list", "_resolve_max_tokens", "len", "_compute_within_policy_timeout", "bool", "parse_status_response", "result.get", "_build_tool_calls_summary" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "record" ], "awaits": false, "yields": false }, { "name": "run_workload_case", "qualname": "run_workload_case", "full_name": "vllm_mlx.bench_serve.run_workload_case", "kind": "function", "signature": "async def run_workload_case(client: httpx.AsyncClient, base_url: str, *, workload: Workload, case: WorkloadCase, model: str, runtime: dict, hardware: dict, run_id: str, timestamp: str, repetition: int=0, scrape: bool=True, include_content: bool=False, cache_reset: Optional[dict]=None) -> dict", "parameters": [ { "name": "client", "kind": "positional or keyword", "annotation": "httpx.AsyncClient", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "base_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "workload", "kind": "keyword-only", "annotation": "Workload", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "case", "kind": "keyword-only", "annotation": "WorkloadCase", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "model", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "runtime", "kind": "keyword-only", "annotation": "dict", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "hardware", "kind": "keyword-only", "annotation": "dict", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "run_id", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "timestamp", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "repetition", "kind": "keyword-only", "annotation": "int", "default": "0", "required": false, "description": "Optional keyword-only input; defaults to `0`." }, { "name": "scrape", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." }, { "name": "include_content", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." }, { "name": "cache_reset", "kind": "keyword-only", "annotation": "Optional[dict]", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "dict", "docstring": "Run one workload case and return a JSON-serializable result.", "summary": "Run one workload case and return a JSON-serializable result.", "implementation": "Function `run_workload_case` calls `scrape_metrics`, `datetime.now(timezone.utc).isoformat`, `datetime.now`, `_assemble_case_request_kwargs`; awaits asynchronous work; returns `_build_workload_record(case=case, workload=workload, model=model, runtime=runtime, hardware=hardware, run_id=run_id, ti…`.", "documented": true, "public": true, "addressable": true, "line": 1518, "end_line": 1593, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1518-L1593", "decorators": [], "calls": [ "scrape_metrics", "datetime.now(timezone.utc).isoformat", "datetime.now", "_assemble_case_request_kwargs", "stream_chat_completion", "_empty_completion_result", "str", "_fetch_post_run_status", "metrics_after.get", "metrics_before.get", "result.get", "validate_quality_checks", "quality_issues.append", "_build_workload_record" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_build_workload_record(case=case, workload=workload, model=model, runtime=runtime, hardware=hardware, run_id=run_id, ti…" ], "awaits": true, "yields": false }, { "name": "_group_results_by_case_id", "qualname": "_group_results_by_case_id", "full_name": "vllm_mlx.bench_serve._group_results_by_case_id", "kind": "function", "signature": "def _group_results_by_case_id(results: list[dict]) -> dict[str, list[dict]]", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, list[dict]]", "docstring": "Bucket workload case records by their ``case_id`` field, defaulting\na missing ``case_id`` to the empty string so the grouping is stable.", "summary": "Bucket workload case records by their ``case_id`` field, defaulting a missing ``case_id`` to the empty string so the grouping is stable.", "implementation": "Function `_group_results_by_case_id` calls `cases.setdefault(str(result.get('case_id', '')), []).append`, `cases.setdefault`, `str`, `result.get`; returns `cases`.", "documented": true, "public": false, "addressable": true, "line": 1596, "end_line": 1602, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1596-L1602", "decorators": [], "calls": [ "cases.setdefault(str(result.get('case_id', '')), []).append", "cases.setdefault", "str", "result.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cases" ], "awaits": false, "yields": false }, { "name": "_summarize_case", "qualname": "_summarize_case", "full_name": "vllm_mlx.bench_serve._summarize_case", "kind": "function", "signature": "def _summarize_case(case_results: list[dict]) -> dict", "parameters": [ { "name": "case_results", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Build the per-case summary block.\n\nMirrors the shape used at the run-level (sample counts, pass/fail\nrates, policy-timeout outcome, latency / ttft / gen_tps summaries)\nand adds two case-only fields: ``sample_count`` and ``repetitions``\n(the sorted set of repetition indices the case was run under), plus\n``content_chars`` since content length is more useful per-case than\nper-run.", "summary": "Build the per-case summary block.", "implementation": "Function `_summarize_case` calls `r['quality'].get`, `r['policy'].get`, `len`, `sorted`; returns `{'sample_count': len(case_results), 'repetitions': sorted({int(r.get('repetition', 0)) for r in case_results if r.get('…`.", "documented": true, "public": false, "addressable": true, "line": 1605, "end_line": 1648, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1605-L1648", "decorators": [], "calls": [ "r['quality'].get", "r['policy'].get", "len", "sorted", "int", "r.get", "round", "_summary_or_empty" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'sample_count': len(case_results), 'repetitions': sorted({int(r.get('repetition', 0)) for r in case_results if r.get('…" ], "awaits": false, "yields": false }, { "name": "summarize_workload_results", "qualname": "summarize_workload_results", "full_name": "vllm_mlx.bench_serve.summarize_workload_results", "kind": "function", "signature": "def summarize_workload_results(results: list[dict]) -> dict", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Aggregate workload case records into stable qualification summary stats.", "summary": "Aggregate workload case records into stable qualification summary stats.", "implementation": "Function `summarize_workload_results` calls `r['policy'].get`, `_group_results_by_case_id`, `_summarize_case`, `sorted`; returns `{'case_count': len(results), 'unique_case_count': len(cases), 'repetition_count': max((len(summary['repetitions']) for …`.", "documented": true, "public": true, "addressable": true, "line": 1651, "end_line": 1689, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1651-L1689", "decorators": [], "calls": [ "r['policy'].get", "_group_results_by_case_id", "_summarize_case", "sorted", "cases.items", "len", "max", "case_summaries.values", "round", "_summary_or_empty" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'case_count': len(results), 'unique_case_count': len(cases), 'repetition_count': max((len(summary['repetitions']) for …" ], "awaits": false, "yields": false }, { "name": "run_bench_serve_workload", "qualname": "run_bench_serve_workload", "full_name": "vllm_mlx.bench_serve.run_bench_serve_workload", "kind": "function", "signature": "async def run_bench_serve_workload(*, url: str, workload_path: str, model: Optional[str]=None, output_path: Optional[str]=None, output_format: str='json', scrape: bool=True, include_content: bool=False, request_timeout_s: Optional[float]=300.0, repetitions: int=1, cache_policy: Optional[str]=None) -> dict", "parameters": [ { "name": "url", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "workload_path", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "model", "kind": "keyword-only", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "output_path", "kind": "keyword-only", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "output_format", "kind": "keyword-only", "annotation": "str", "default": "'json'", "required": false, "description": "Optional keyword-only input; defaults to `'json'`." }, { "name": "scrape", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." }, { "name": "include_content", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." }, { "name": "request_timeout_s", "kind": "keyword-only", "annotation": "Optional[float]", "default": "300.0", "required": false, "description": "Optional keyword-only input; defaults to `300.0`." }, { "name": "repetitions", "kind": "keyword-only", "annotation": "int", "default": "1", "required": false, "description": "Optional keyword-only input; defaults to `1`." }, { "name": "cache_policy", "kind": "keyword-only", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "dict", "docstring": "Run a declarative workload against a running server.\n\nThis is the contract-style counterpart to prompt sweeps: it keeps product\npolicy knobs in the manifest, records them as evidence, and measures what\nthe server actually does before anyone promotes a model or feature stack.", "summary": "Run a declarative workload against a running server.", "implementation": "Function `run_bench_serve_workload` calls `ValueError`, `load_workload`, `_normalize_cache_policy`, `workload.defaults.get`; awaits asynchronous work; can raise `ValueError`; returns `payload`.", "documented": true, "public": true, "addressable": true, "line": 1692, "end_line": 1818, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1692-L1818", "decorators": [], "calls": [ "ValueError", "load_workload", "_normalize_cache_policy", "workload.defaults.get", "str", "uuid.uuid4", "datetime.now(timezone.utc).isoformat", "datetime.now", "httpx.Timeout", "httpx.AsyncClient", "auto_detect_runtime", "detect_hardware_fingerprint", "runtime.get", "cache_events.append", "clear_runtime_cache", "len", "range", "print", "run_workload_case", "records.append", "record.get('metrics', {}).get", "record.get", "Path(workload_path).expanduser", "Path", "summarize_workload_results", "write_workload_sqlite", "format_workload_payload", "Path(output_path).expanduser().write_text", "Path(output_path).expanduser" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "payload" ], "awaits": true, "yields": false }, { "name": "_result_to_dict", "qualname": "_result_to_dict", "full_name": "vllm_mlx.bench_serve._result_to_dict", "kind": "function", "signature": "def _result_to_dict(r: BenchServeResult) -> dict", "parameters": [ { "name": "r", "kind": "positional or keyword", "annotation": "BenchServeResult", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Convert a :class:`BenchServeResult` to an ordered dict.\n\nReturns an ``OrderedDict``-style plain ``dict`` whose keys follow the\ndataclass field declaration order (as listed in :data:`RESULT_COLUMNS`).", "summary": "Convert a :class:`BenchServeResult` to an ordered dict.", "implementation": "Function `_result_to_dict` calls `getattr`, `_dataclasses.fields`; returns `{f.name: getattr(r, f.name) for f in _dataclasses.fields(r)}`.", "documented": true, "public": false, "addressable": true, "line": 1840, "end_line": 1846, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1840-L1846", "decorators": [], "calls": [ "getattr", "_dataclasses.fields" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{f.name: getattr(r, f.name) for f in _dataclasses.fields(r)}" ], "awaits": false, "yields": false }, { "name": "format_table", "qualname": "format_table", "full_name": "vllm_mlx.bench_serve.format_table", "kind": "function", "signature": "def format_table(results: list[BenchServeResult]) -> str", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[BenchServeResult]", "default": "", "required": true, "description": "List of :class:`BenchServeResult` instances." } ], "return_annotation": "str", "docstring": "Render a human-readable terminal table of benchmark results.\n\nOnly the columns in :data:`_TABLE_COLUMNS` are shown. Float values are\nrounded to one decimal place.\n\nArgs:\n results: List of :class:`BenchServeResult` instances.\n\nReturns:\n Formatted string using ``tabulate`` with ``tablefmt=\"simple\"``.", "summary": "Render a human-readable terminal table of benchmark results.", "implementation": "Function `format_table` calls `_result_to_dict`, `d.get`, `isinstance`, `round`; returns `_tabulate(rows, headers=_TABLE_COLUMNS, tablefmt='simple')`.", "documented": true, "public": true, "addressable": true, "line": 1849, "end_line": 1871, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1849-L1871", "decorators": [], "calls": [ "_result_to_dict", "d.get", "isinstance", "round", "row.append", "rows.append", "_tabulate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_tabulate(rows, headers=_TABLE_COLUMNS, tablefmt='simple')" ], "awaits": false, "yields": false }, { "name": "format_json", "qualname": "format_json", "full_name": "vllm_mlx.bench_serve.format_json", "kind": "function", "signature": "def format_json(results: list[BenchServeResult]) -> str", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[BenchServeResult]", "default": "", "required": true, "description": "List of :class:`BenchServeResult` instances." } ], "return_annotation": "str", "docstring": "Serialize benchmark results as a JSON array.\n\nAll fields from :data:`RESULT_COLUMNS` are included.\n\nArgs:\n results: List of :class:`BenchServeResult` instances.\n\nReturns:\n JSON string with ``indent=2``.", "summary": "Serialize benchmark results as a JSON array.", "implementation": "Function `format_json` calls `json.dumps`, `_result_to_dict`; returns `json.dumps([_result_to_dict(r) for r in results], indent=2)`.", "documented": true, "public": true, "addressable": true, "line": 1874, "end_line": 1885, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1874-L1885", "decorators": [], "calls": [ "json.dumps", "_result_to_dict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.dumps([_result_to_dict(r) for r in results], indent=2)" ], "awaits": false, "yields": false }, { "name": "format_csv", "qualname": "format_csv", "full_name": "vllm_mlx.bench_serve.format_csv", "kind": "function", "signature": "def format_csv(results: list[BenchServeResult]) -> str", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[BenchServeResult]", "default": "", "required": true, "description": "List of :class:`BenchServeResult` instances." } ], "return_annotation": "str", "docstring": "Serialize benchmark results as CSV with a header row.\n\nAll columns are included.\n\nArgs:\n results: List of :class:`BenchServeResult` instances.\n\nReturns:\n CSV string (header + one row per result).", "summary": "Serialize benchmark results as CSV with a header row.", "implementation": "Function `format_csv` calls `io.StringIO`, `csv_mod.DictWriter`, `writer.writeheader`, `writer.writerow`; returns `buf.getvalue()`.", "documented": true, "public": true, "addressable": true, "line": 1888, "end_line": 1904, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1888-L1904", "decorators": [], "calls": [ "io.StringIO", "csv_mod.DictWriter", "writer.writeheader", "writer.writerow", "_result_to_dict", "buf.getvalue" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "buf.getvalue()" ], "awaits": false, "yields": false }, { "name": "_sql_escape", "qualname": "_sql_escape", "full_name": "vllm_mlx.bench_serve._sql_escape", "kind": "function", "signature": "def _sql_escape(value) -> str", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Escape a Python value for use as a SQL literal.\n\n- ``None`` -> ``\"NULL\"``\n- ``bool`` -> ``\"1\"`` or ``\"0\"``\n- ``int`` / ``float`` -> string representation\n- ``str`` -> single-quoted with internal single-quotes doubled", "summary": "Escape a Python value for use as a SQL literal.", "implementation": "Function `_sql_escape` calls `isinstance`, `math.isnan`, `math.isinf`, `str`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1907, "end_line": 1927, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1907-L1927", "decorators": [], "calls": [ "isinstance", "math.isnan", "math.isinf", "str", "str(value).replace" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'NULL'", "'1' if value else '0'", "str(value)", "f\"'{escaped}'\"" ], "awaits": false, "yields": false }, { "name": "format_sql", "qualname": "format_sql", "full_name": "vllm_mlx.bench_serve.format_sql", "kind": "function", "signature": "def format_sql(results: list[BenchServeResult]) -> str", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[BenchServeResult]", "default": "", "required": true, "description": "List of :class:`BenchServeResult` instances." } ], "return_annotation": "str", "docstring": "Emit a SQL ``CREATE TABLE IF NOT EXISTS`` statement and INSERT rows.\n\nThe schema follows the exact column order defined in the bench-serve spec.\n\nArgs:\n results: List of :class:`BenchServeResult` instances.\n\nReturns:\n SQL string containing the CREATE TABLE statement followed by one\n INSERT statement per result.", "summary": "Emit a SQL ``CREATE TABLE IF NOT EXISTS`` statement and INSERT rows.", "implementation": "Function `format_sql` calls `_result_to_dict`, `', '.join`, `_sql_escape`, `lines.append`; returns `'\\n'.join(lines)`.", "documented": true, "public": true, "addressable": true, "line": 1945, "end_line": 1964, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1945-L1964", "decorators": [], "calls": [ "_result_to_dict", "', '.join", "_sql_escape", "lines.append", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'\\n'.join(lines)" ], "awaits": false, "yields": false }, { "name": "_write_sqlite_rows", "qualname": "_write_sqlite_rows", "full_name": "vllm_mlx.bench_serve._write_sqlite_rows", "kind": "function", "signature": "def _write_sqlite_rows(output_path: str, *, table: str, schema: str, columns: list[str], rows: list[dict]) -> None", "parameters": [ { "name": "output_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "table", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "schema", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "columns", "kind": "keyword-only", "annotation": "list[str]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "rows", "kind": "keyword-only", "annotation": "list[dict]", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Append benchmark rows to a SQLite database.", "summary": "Append benchmark rows to a SQLite database.", "implementation": "Function `_write_sqlite_rows` calls `Path(output_path).expanduser`, `Path`, `_validate_sql_identifier`, `', '.join`.", "documented": true, "public": false, "addressable": true, "line": 1967, "end_line": 1990, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1967-L1990", "decorators": [], "calls": [ "Path(output_path).expanduser", "Path", "_validate_sql_identifier", "', '.join", "row.get", "sqlite3.connect", "conn.execute", "conn.executemany", "conn.commit" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_validate_sql_identifier", "qualname": "_validate_sql_identifier", "full_name": "vllm_mlx.bench_serve._validate_sql_identifier", "kind": "function", "signature": "def _validate_sql_identifier(identifier: str, *, kind: str) -> None", "parameters": [ { "name": "identifier", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "kind", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Reject unsafe SQL identifiers before string interpolation.", "summary": "Reject unsafe SQL identifiers before string interpolation.", "implementation": "Function `_validate_sql_identifier` calls `_SQL_IDENTIFIER_RE.fullmatch`, `ValueError`; can raise `ValueError`.", "documented": true, "public": false, "addressable": true, "line": 1993, "end_line": 1996, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1993-L1996", "decorators": [], "calls": [ "_SQL_IDENTIFIER_RE.fullmatch", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "write_sqlite", "qualname": "write_sqlite", "full_name": "vllm_mlx.bench_serve.write_sqlite", "kind": "function", "signature": "def write_sqlite(results: list[BenchServeResult], output_path: str) -> None", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[BenchServeResult]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "output_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Append prompt-sweep benchmark results to a SQLite database.", "summary": "Append prompt-sweep benchmark results to a SQLite database.", "implementation": "Function `write_sqlite` calls `_result_to_dict`, `_write_sqlite_rows`.", "documented": true, "public": true, "addressable": true, "line": 1999, "end_line": 2009, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L1999-L2009", "decorators": [], "calls": [ "_result_to_dict", "_write_sqlite_rows" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_workload_record_to_row", "qualname": "_workload_record_to_row", "full_name": "vllm_mlx.bench_serve._workload_record_to_row", "kind": "function", "signature": "def _workload_record_to_row(record: dict) -> dict", "parameters": [ { "name": "record", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "", "summary": "Function `_workload_record_to_row` calls `record.get`, `metrics.get`, `','.join`, `hardware.get`; returns `{'run_id': record.get('run_id', ''), 'timestamp': record.get('timestamp', ''), 'workload': record.get('workload', ''), …`.", "implementation": "Function `_workload_record_to_row` calls `record.get`, `metrics.get`, `','.join`, `hardware.get`; returns `{'run_id': record.get('run_id', ''), 'timestamp': record.get('timestamp', ''), 'workload': record.get('workload', ''), …`.", "documented": false, "public": false, "addressable": true, "line": 2069, "end_line": 2119, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2069-L2119", "decorators": [], "calls": [ "record.get", "metrics.get", "','.join", "hardware.get", "runtime.get", "request.get", "json.dumps", "policy.get", "metal.get", "quality.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'run_id': record.get('run_id', ''), 'timestamp': record.get('timestamp', ''), 'workload': record.get('workload', ''), …" ], "awaits": false, "yields": false }, { "name": "format_workload_table", "qualname": "format_workload_table", "full_name": "vllm_mlx.bench_serve.format_workload_table", "kind": "function", "signature": "def format_workload_table(payload: dict) -> str", "parameters": [ { "name": "payload", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Format workload result records as a compact human-readable table.", "summary": "Format workload result records as a compact human-readable table.", "implementation": "Function `format_workload_table` calls `payload.get`, `_workload_record_to_row`, `rows.append`, `isinstance`; returns `_tabulate(rows, headers=_WORKLOAD_TABLE_COLUMNS, tablefmt='simple')`.", "documented": true, "public": true, "addressable": true, "line": 2122, "end_line": 2134, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2122-L2134", "decorators": [], "calls": [ "payload.get", "_workload_record_to_row", "rows.append", "isinstance", "round", "_tabulate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_tabulate(rows, headers=_WORKLOAD_TABLE_COLUMNS, tablefmt='simple')" ], "awaits": false, "yields": false }, { "name": "format_workload_json", "qualname": "format_workload_json", "full_name": "vllm_mlx.bench_serve.format_workload_json", "kind": "function", "signature": "def format_workload_json(payload: dict) -> str", "parameters": [ { "name": "payload", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Serialize a workload result payload as indented JSON.", "summary": "Serialize a workload result payload as indented JSON.", "implementation": "Function `format_workload_json` calls `json.dumps`; returns `json.dumps(payload, indent=2)`.", "documented": true, "public": true, "addressable": true, "line": 2137, "end_line": 2140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2137-L2140", "decorators": [], "calls": [ "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.dumps(payload, indent=2)" ], "awaits": false, "yields": false }, { "name": "format_workload_csv", "qualname": "format_workload_csv", "full_name": "vllm_mlx.bench_serve.format_workload_csv", "kind": "function", "signature": "def format_workload_csv(payload: dict) -> str", "parameters": [ { "name": "payload", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Serialize workload result records with the stable CSV column contract.", "summary": "Serialize workload result records with the stable CSV column contract.", "implementation": "Function `format_workload_csv` calls `io.StringIO`, `csv_mod.DictWriter`, `writer.writeheader`, `payload.get`; returns `buf.getvalue()`.", "documented": true, "public": true, "addressable": true, "line": 2143, "end_line": 2151, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2143-L2151", "decorators": [], "calls": [ "io.StringIO", "csv_mod.DictWriter", "writer.writeheader", "payload.get", "writer.writerow", "_workload_record_to_row", "buf.getvalue" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "buf.getvalue()" ], "awaits": false, "yields": false }, { "name": "format_workload_sql", "qualname": "format_workload_sql", "full_name": "vllm_mlx.bench_serve.format_workload_sql", "kind": "function", "signature": "def format_workload_sql(payload: dict) -> str", "parameters": [ { "name": "payload", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Render SQL statements that create and populate the workload table.", "summary": "Render SQL statements that create and populate the workload table.", "implementation": "Function `format_workload_sql` calls `payload.get`, `_workload_record_to_row`, `', '.join`, `_sql_escape`; returns `'\\n'.join(lines)`.", "documented": true, "public": true, "addressable": true, "line": 2170, "end_line": 2180, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2170-L2180", "decorators": [], "calls": [ "payload.get", "_workload_record_to_row", "', '.join", "_sql_escape", "lines.append", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'\\n'.join(lines)" ], "awaits": false, "yields": false }, { "name": "write_workload_sqlite", "qualname": "write_workload_sqlite", "full_name": "vllm_mlx.bench_serve.write_workload_sqlite", "kind": "function", "signature": "def write_workload_sqlite(payload: dict, output_path: str) -> None", "parameters": [ { "name": "payload", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "output_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Append workload result records to a SQLite database.", "summary": "Append workload result records to a SQLite database.", "implementation": "Function `write_workload_sqlite` calls `_workload_record_to_row`, `payload.get`, `_write_sqlite_rows`.", "documented": true, "public": true, "addressable": true, "line": 2183, "end_line": 2193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2183-L2193", "decorators": [], "calls": [ "_workload_record_to_row", "payload.get", "_write_sqlite_rows" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "format_workload_payload", "qualname": "format_workload_payload", "full_name": "vllm_mlx.bench_serve.format_workload_payload", "kind": "function", "signature": "def format_workload_payload(payload: dict, fmt: str='json') -> str", "parameters": [ { "name": "payload", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "fmt", "kind": "positional or keyword", "annotation": "str", "default": "'json'", "required": false, "description": "Optional positional or keyword input; defaults to `'json'`." } ], "return_annotation": "str", "docstring": "Serialize a workload payload in the requested text output format.\n\nRaises:\n ValueError: If ``fmt`` is not ``json``, ``csv``, ``sql``, or ``table``.", "summary": "Serialize a workload payload in the requested text output format.", "implementation": "Function `format_workload_payload` calls `format_workload_json`, `format_workload_csv`, `format_workload_sql`, `format_workload_table`; can raise `ValueError`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 2196, "end_line": 2211, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2196-L2211", "decorators": [], "calls": [ "format_workload_json", "format_workload_csv", "format_workload_sql", "format_workload_table", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "format_workload_json(payload)", "format_workload_csv(payload)", "format_workload_sql(payload)", "format_workload_table(payload)" ], "awaits": false, "yields": false }, { "name": "run_bench_serve", "qualname": "run_bench_serve", "full_name": "vllm_mlx.bench_serve.run_bench_serve", "kind": "function", "signature": "async def run_bench_serve(url: str='http://127.0.0.1:8080', model: Optional[str]=None, prompt_sets: list[str]=None, prompt_file: Optional[str]=None, concurrencies: list[int]=None, max_tokens: int=256, repetitions: int=3, warmup: int=1, thinking_values: list[Optional[bool]]=None, extra_bodies: list[str]=None, output_path: Optional[str]=None, fmt: str='table', do_validate: bool=True, scrape: bool=True, tag: Optional[str]=None, override_fields: Optional[dict]=None, system_prompt_file: Optional[str]=None, skip_preflight_token_count: bool=False) -> list[BenchServeResult]", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "'http://127.0.0.1:8080'", "required": false, "description": "Base URL of the server." }, { "name": "model", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Model ID to use. If ``None``, auto-detected from the server." }, { "name": "prompt_sets", "kind": "positional or keyword", "annotation": "list[str]", "default": "None", "required": false, "description": "List of prompt set names or paths. Defaults to ``[\"short\", \"medium\", \"long\"]``." }, { "name": "prompt_file", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional path to an extra prompt file to include." }, { "name": "concurrencies", "kind": "positional or keyword", "annotation": "list[int]", "default": "None", "required": false, "description": "Concurrency levels to sweep. Defaults to ``[1, 4]``." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate per request." }, { "name": "repetitions", "kind": "positional or keyword", "annotation": "int", "default": "3", "required": false, "description": "Number of repetitions per sweep config." }, { "name": "warmup", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Number of warmup rounds before the first measured repetition." }, { "name": "thinking_values", "kind": "positional or keyword", "annotation": "list[Optional[bool]]", "default": "None", "required": false, "description": "Values for ``enable_thinking``. Defaults to ``[None]``." }, { "name": "extra_bodies", "kind": "positional or keyword", "annotation": "list[str]", "default": "None", "required": false, "description": "JSON strings for extra body parameters. Defaults to ``[\"\"]`` (no extra body)." }, { "name": "output_path", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "File path to write results to. If ``None``, prints to stdout." }, { "name": "fmt", "kind": "positional or keyword", "annotation": "str", "default": "'table'", "required": false, "description": "Output format — one of ``\"table\"``, ``\"json\"``, ``\"csv\"``, ``\"sql\"``, or ``\"sqlite\"``." }, { "name": "do_validate", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether to validate each response." }, { "name": "scrape", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether to scrape ``/metrics`` before and after each run." }, { "name": "tag", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional tag string stored in every result row." }, { "name": "override_fields", "kind": "positional or keyword", "annotation": "Optional[dict]", "default": "None", "required": false, "description": "Dict of field names to override on every result." }, { "name": "system_prompt_file", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "skip_preflight_token_count", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "list[BenchServeResult]", "docstring": "Run the full bench-serve sweep against a running vllm-mlx server.\n\nArgs:\n url: Base URL of the server.\n model: Model ID to use. If ``None``, auto-detected from the server.\n prompt_sets: List of prompt set names or paths. Defaults to\n ``[\"short\", \"medium\", \"long\"]``.\n prompt_file: Optional path to an extra prompt file to include.\n concurrencies: Concurrency levels to sweep. Defaults to ``[1, 4]``.\n max_tokens: Maximum tokens to generate per request.\n repetitions: Number of repetitions per sweep config.\n warmup: Number of warmup rounds before the first measured repetition.\n thinking_values: Values for ``enable_thinking``. Defaults to\n ``[None]``.\n extra_bodies: JSON strings for extra body parameters. Defaults to\n ``[\"\"]`` (no extra body).\n output_path: File path to write results to. If ``None``, prints to\n stdout.\n fmt: Output format — one of ``\"table\"``, ``\"json\"``, ``\"csv\"``,\n ``\"sql\"``, or ``\"sqlite\"``.\n do_validate: Whether to validate each response.\n scrape: Whether to scrape ``/metrics`` before and after each run.\n tag: Optional tag string stored in every result row.\n override_fields: Dict of field names to override on every result.\n\nReturns:\n List of :class:`BenchServeResult` instances.", "summary": "Run the full bench-serve sweep against a running vllm-mlx server.", "implementation": "Function `run_bench_serve` calls `str`, `uuid.uuid4`, `datetime.now(timezone.utc).isoformat`, `datetime.now`; awaits asynchronous work; can raise `ValueError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 2221, "end_line": 2638, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2221-L2638", "decorators": [], "calls": [ "str", "uuid.uuid4", "datetime.now(timezone.utc).isoformat", "datetime.now", "httpx.AsyncClient", "httpx.Timeout", "print", "auto_detect_runtime", "detect_hardware_fingerprint", "runtime.get", "hw.get", "load_prompt_set", "Path(system_prompt_file).expanduser", "Path", "sys_path.exists", "sys_path.read_text", "all_prompts.items", "msgs[0].get", "patched.append", "len", "count_prompt_tokens", "expand_sweep", "list", "all_prompts.keys", "set", "json.loads", "label_parts.append", "' '.join", "warmed_up.add", "range", "run_concurrent_requests", "scrape_metrics", "metrics_after.get", "metrics_before.get", "client.get", "resp.raise_for_status", "parse_status_response", "resp.json", "status_data.get", "BenchServeResult", "prompt_token_counts.get", "_mean", "sum", "r.get", "max", "all", "override_fields.items", "hasattr", "setattr", "results.append", "ValueError", "write_sqlite", "formatters.get", "formatter", "Path(output_path).write_text" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "[]", "results" ], "awaits": true, "yields": false }, { "name": "_mean", "qualname": "run_bench_serve._mean", "full_name": "vllm_mlx.bench_serve.run_bench_serve._mean", "kind": "nested function", "signature": "def _mean(key: str) -> float", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "", "summary": "Nested Function `run_bench_serve._mean` calls `statistics.mean`; returns `statistics.mean(vals) if vals else 0.0`.", "implementation": "Nested Function `run_bench_serve._mean` calls `statistics.mean`; returns `statistics.mean(vals) if vals else 0.0`.", "documented": false, "public": false, "addressable": false, "line": 2522, "end_line": 2526, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/bench_serve.py#L2522-L2526", "decorators": [], "calls": [ "statistics.mean" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "statistics.mean(vals) if vals else 0.0" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.benchmark", "path": "vllm_mlx/benchmark.py", "page_path": "reference/api/vllm_mlx/benchmark.md", "docstring": "Performance Benchmark for vllm-mlx.\n\nMeasures key performance metrics for LLM and MLLM (Multimodal Language Model) inference:\n- Time to First Token (TTFT)\n- Time Per Output Token (TPOT)\n- Tokens Per Second (TPS) - both input processing and output generation\n- End-to-End Latency\n- Throughput\n- Memory Usage (process and MLX cache)\n- MLLM: Image resolution performance\n- MLLM: Video frame count performance\n\nUsage:\n # LLM benchmark\n python -m vllm_mlx.benchmark --model mlx-community/Llama-3.2-1B-Instruct-4bit\n python -m vllm_mlx.benchmark --model mlx-community/Llama-3.2-3B-Instruct-4bit --prompts 10 --max-tokens 256\n\n # MLLM image benchmark (auto-detected or use --mllm flag)\n python -m vllm_mlx.benchmark --model mlx-community/Qwen3-VL-4B-Instruct-3bit\n python -m vllm_mlx.benchmark --model mlx-community/Qwen3-VL-4B-Instruct-3bit --mllm --quick\n\n # MLLM video benchmark\n python -m vllm_mlx.benchmark --model mlx-community/Qwen3-VL-4B-Instruct-3bit --video\n python -m vllm_mlx.benchmark --model mlx-community/Qwen3-VL-4B-Instruct-3bit --video --video-url https://example.com/video.mp4", "summary": "Performance Benchmark for vllm-mlx.", "line_count": 1684, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1-L1684", "members": [ "ResourceMetrics", "reset_mlx_peak_memory", "get_mlx_memory_info", "get_process_memory", "get_system_memory", "ResourceMonitor", "VIDEO_SAMPLE_URLS", "DEFAULT_VIDEO_URL", "VLM_TEST_VIDEO_URLS", "BenchmarkResult", "BenchmarkSummary", "calculate_percentile", "benchmark_single_prompt", "run_benchmark", "MLLM_PATTERNS", "MLLM_TEST_IMAGE_URL", "MLLM_TEST_IMAGE_URLS", "is_mllm_model", "MLLMBenchmarkResult", "download_test_image", "resize_image", "image_to_base64", "benchmark_mllm_resolution", "run_mllm_benchmark", "print_mllm_summary", "VideoBenchmarkResult", "create_test_video", "download_video", "get_video_info", "benchmark_video_config", "run_video_benchmark", "print_video_summary", "print_summary", "main" ], "symbols": [ { "name": "ResourceMetrics", "qualname": "ResourceMetrics", "full_name": "vllm_mlx.benchmark.ResourceMetrics", "kind": "class", "signature": "class ResourceMetrics", "parameters": [ { "name": "process_memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "mlx_cache_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "mlx_peak_memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "system_memory_used_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "system_memory_total_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "ResourceMetrics", "docstring": "Resource usage metrics during benchmark.", "summary": "Resource usage metrics during benchmark.", "implementation": "Class `ResourceMetrics` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 72, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L72-L80", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset_mlx_peak_memory", "qualname": "reset_mlx_peak_memory", "full_name": "vllm_mlx.benchmark.reset_mlx_peak_memory", "kind": "function", "signature": "def reset_mlx_peak_memory()", "parameters": [], "return_annotation": "not annotated", "docstring": "Reset MLX peak memory counter.", "summary": "Reset MLX peak memory counter.", "implementation": "Function `reset_mlx_peak_memory` calls `hasattr`, `mx.reset_peak_memory`, `mx.metal.reset_peak_memory`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 83, "end_line": 95, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L83-L95", "decorators": [], "calls": [ "hasattr", "mx.reset_peak_memory", "mx.metal.reset_peak_memory" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_mlx_memory_info", "qualname": "get_mlx_memory_info", "full_name": "vllm_mlx.benchmark.get_mlx_memory_info", "kind": "function", "signature": "def get_mlx_memory_info(reset_peak: bool=True) -> dict", "parameters": [ { "name": "reset_peak", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, reset peak memory counter after reading." } ], "return_annotation": "dict", "docstring": "Get MLX memory usage information.\n\nArgs:\n reset_peak: If True, reset peak memory counter after reading.", "summary": "Get MLX memory usage information.", "implementation": "Function `get_mlx_memory_info` calls `hasattr`, `mx.get_cache_memory`, `mx.get_peak_memory`, `mx.get_active_memory`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 98, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L98-L135", "decorators": [], "calls": [ "hasattr", "mx.get_cache_memory", "mx.get_peak_memory", "mx.get_active_memory", "mx.metal.get_cache_memory", "mx.metal.get_peak_memory", "mx.metal.get_active_memory", "reset_mlx_peak_memory" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{}", "info" ], "awaits": false, "yields": false }, { "name": "get_process_memory", "qualname": "get_process_memory", "full_name": "vllm_mlx.benchmark.get_process_memory", "kind": "function", "signature": "def get_process_memory() -> float", "parameters": [], "return_annotation": "float", "docstring": "Get current process memory usage in GB.", "summary": "Get current process memory usage in GB.", "implementation": "Function `get_process_memory` calls `psutil.Process`, `process.memory_info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 138, "end_line": 147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L138-L147", "decorators": [], "calls": [ "psutil.Process", "process.memory_info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0.0", "process.memory_info().rss / 1024 ** 3" ], "awaits": false, "yields": false }, { "name": "get_system_memory", "qualname": "get_system_memory", "full_name": "vllm_mlx.benchmark.get_system_memory", "kind": "function", "signature": "def get_system_memory() -> tuple[float, float]", "parameters": [], "return_annotation": "tuple[float, float]", "docstring": "Get system memory (used, total) in GB.", "summary": "Get system memory (used, total) in GB.", "implementation": "Function `get_system_memory` calls `psutil.virtual_memory`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 150, "end_line": 159, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L150-L159", "decorators": [], "calls": [ "psutil.virtual_memory" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(0.0, 0.0)", "(mem.used / 1024 ** 3, mem.total / 1024 ** 3)" ], "awaits": false, "yields": false }, { "name": "ResourceMonitor", "qualname": "ResourceMonitor", "full_name": "vllm_mlx.benchmark.ResourceMonitor", "kind": "class", "signature": "class ResourceMonitor", "parameters": [], "return_annotation": "ResourceMonitor", "docstring": "Monitor system resources during benchmark runs.", "summary": "Monitor system resources during benchmark runs.", "implementation": "Class `ResourceMonitor` declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 162, "end_line": 213, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L162-L213", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ResourceMonitor.__init__", "full_name": "vllm_mlx.benchmark.ResourceMonitor.__init__", "kind": "method", "signature": "def __init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `ResourceMonitor.__init__` updates `self.samples`, `self._start_time`, `self._start_memory`.", "implementation": "Method `ResourceMonitor.__init__` updates `self.samples`, `self._start_time`, `self._start_memory`.", "documented": false, "public": false, "addressable": true, "line": 165, "end_line": 168, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L165-L168", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.samples", "self._start_time", "self._start_memory" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start", "qualname": "ResourceMonitor.start", "full_name": "vllm_mlx.benchmark.ResourceMonitor.start", "kind": "method", "signature": "def start(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Start monitoring.", "summary": "Start monitoring.", "implementation": "Method `ResourceMonitor.start` updates `self._start_time`, `self._start_memory`; calls `time.perf_counter`, `get_process_memory`, `reset_mlx_peak_memory`.", "documented": true, "public": true, "addressable": true, "line": 170, "end_line": 176, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L170-L176", "decorators": [], "calls": [ "time.perf_counter", "get_process_memory", "reset_mlx_peak_memory" ], "state_reads": [], "state_writes": [ "self._start_time", "self._start_memory" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "sample", "qualname": "ResourceMonitor.sample", "full_name": "vllm_mlx.benchmark.ResourceMonitor.sample", "kind": "method", "signature": "def sample(self) -> ResourceMetrics", "parameters": [], "return_annotation": "ResourceMetrics", "docstring": "Take a resource sample.", "summary": "Take a resource sample.", "implementation": "Method `ResourceMonitor.sample` calls `get_mlx_memory_info`, `get_system_memory`, `ResourceMetrics`, `get_process_memory`; returns `metrics`.", "documented": true, "public": true, "addressable": true, "line": 178, "end_line": 192, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L178-L192", "decorators": [], "calls": [ "get_mlx_memory_info", "get_system_memory", "ResourceMetrics", "get_process_memory", "mlx_info.get", "self.samples.append" ], "state_reads": [ "self.samples.append", "self.samples" ], "state_writes": [], "raises": [], "return_expressions": [ "metrics" ], "awaits": false, "yields": false }, { "name": "get_summary", "qualname": "ResourceMonitor.get_summary", "full_name": "vllm_mlx.benchmark.ResourceMonitor.get_summary", "kind": "method", "signature": "def get_summary(self) -> ResourceMetrics", "parameters": [], "return_annotation": "ResourceMetrics", "docstring": "Get summary of all samples.", "summary": "Get summary of all samples.", "implementation": "Method `ResourceMonitor.get_summary` calls `ResourceMetrics`, `max`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 194, "end_line": 213, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L194-L213", "decorators": [], "calls": [ "ResourceMetrics", "max" ], "state_reads": [ "self.samples" ], "state_writes": [], "raises": [], "return_expressions": [ "ResourceMetrics()", "ResourceMetrics(process_memory_gb=peak_process, mlx_cache_gb=peak_mlx_cache, mlx_peak_memory_gb=peak_mlx, system_memory…" ], "awaits": false, "yields": false }, { "name": "BenchmarkResult", "qualname": "BenchmarkResult", "full_name": "vllm_mlx.benchmark.BenchmarkResult", "kind": "class", "signature": "class BenchmarkResult", "parameters": [ { "name": "prompt", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "generated_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "ttft", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_time", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tpot", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "generation_tps", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "processing_tps", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "BenchmarkResult", "docstring": "Results from a single benchmark run.", "summary": "Results from a single benchmark run.", "implementation": "Class `BenchmarkResult` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 235, "end_line": 268, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L235-L268", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__post_init__", "qualname": "BenchmarkResult.__post_init__", "full_name": "vllm_mlx.benchmark.BenchmarkResult.__post_init__", "kind": "method", "signature": "def __post_init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `BenchmarkResult.__post_init__` updates `self.tpot`, `self.generation_tps`, `self.processing_tps`.", "implementation": "Method `BenchmarkResult.__post_init__` updates `self.tpot`, `self.generation_tps`, `self.processing_tps`.", "documented": false, "public": false, "addressable": true, "line": 251, "end_line": 268, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L251-L268", "decorators": [], "calls": [], "state_reads": [ "self.generated_tokens", "self.total_time", "self.ttft", "self.prompt_tokens" ], "state_writes": [ "self.tpot", "self.generation_tps", "self.processing_tps" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "BenchmarkSummary", "qualname": "BenchmarkSummary", "full_name": "vllm_mlx.benchmark.BenchmarkSummary", "kind": "class", "signature": "class BenchmarkSummary", "parameters": [ { "name": "model_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "num_runs", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_prompt_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_generated_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_time", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "ttft_mean", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "ttft_min", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "ttft_max", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "ttft_p50", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "ttft_p95", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tpot_mean", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tpot_min", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tpot_max", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "generation_tps_mean", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "generation_tps_max", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "processing_tps_mean", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "latency_mean", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "latency_min", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "latency_max", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "latency_p50", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "latency_p95", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_throughput_tps", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "requests_per_second", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "hardware_chip", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "hardware_memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "hardware_bandwidth_gbs", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "resources", "kind": "field", "annotation": "ResourceMetrics", "default": "field(default_factory=ResourceMetrics)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=ResourceMetrics)`." } ], "return_annotation": "BenchmarkSummary", "docstring": "Summary statistics across all benchmark runs.", "summary": "Summary statistics across all benchmark runs.", "implementation": "Class `BenchmarkSummary` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 272, "end_line": 315, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L272-L315", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "calculate_percentile", "qualname": "calculate_percentile", "full_name": "vllm_mlx.benchmark.calculate_percentile", "kind": "function", "signature": "def calculate_percentile(data: list, percentile: float) -> float", "parameters": [ { "name": "data", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "percentile", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Calculate percentile from a list.", "summary": "Calculate percentile from a list.", "implementation": "Function `calculate_percentile` calls `sorted`, `int`, `len`, `min`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 318, "end_line": 325, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L318-L325", "decorators": [], "calls": [ "sorted", "int", "len", "min" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0.0", "sorted_data[index]" ], "awaits": false, "yields": false }, { "name": "benchmark_single_prompt", "qualname": "benchmark_single_prompt", "full_name": "vllm_mlx.benchmark.benchmark_single_prompt", "kind": "function", "signature": "def benchmark_single_prompt(model, tokenizer, prompt: str, max_tokens: int=256, temperature: float=0.7) -> Optional[BenchmarkResult]", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "The loaded MLX model" }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "The tokenizer" }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The prompt to benchmark" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" } ], "return_annotation": "Optional[BenchmarkResult]", "docstring": "Benchmark a single prompt with detailed timing.\n\nArgs:\n model: The loaded MLX model\n tokenizer: The tokenizer\n prompt: The prompt to benchmark\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n\nReturns:\n BenchmarkResult with timing metrics", "summary": "Benchmark a single prompt with detailed timing.", "implementation": "Function `benchmark_single_prompt` calls `tokenizer.encode`, `len`, `make_sampler`, `time.perf_counter`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 328, "end_line": 394, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L328-L394", "decorators": [], "calls": [ "tokenizer.encode", "len", "make_sampler", "time.perf_counter", "stream_generate", "BenchmarkResult", "print", "traceback.print_exc" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "BenchmarkResult(prompt=prompt[:50] + '...' if len(prompt) > 50 else prompt, prompt_tokens=prompt_token_count, generated…", "None" ], "awaits": false, "yields": false }, { "name": "run_benchmark", "qualname": "run_benchmark", "full_name": "vllm_mlx.benchmark.run_benchmark", "kind": "function", "signature": "def run_benchmark(model_name: str, num_prompts: int=5, max_tokens: int=256, temperature: float=0.7, warmup_runs: int=1) -> Optional[BenchmarkSummary]", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "num_prompts", "kind": "positional or keyword", "annotation": "int", "default": "5", "required": false, "description": "Number of prompts to test" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens per generation" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "warmup_runs", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Number of warmup runs before measuring" } ], "return_annotation": "Optional[BenchmarkSummary]", "docstring": "Run the full benchmark suite.\n\nArgs:\n model_name: HuggingFace model name or local path\n num_prompts: Number of prompts to test\n max_tokens: Maximum tokens per generation\n temperature: Sampling temperature\n warmup_runs: Number of warmup runs before measuring\n\nReturns:\n BenchmarkSummary with aggregate statistics", "summary": "Run the full benchmark suite.", "implementation": "Function `run_benchmark` calls `detect_hardware`, `len`, `print`, `tabulate`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 397, "end_line": 610, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L397-L610", "decorators": [], "calls": [ "detect_hardware", "len", "print", "tabulate", "ResourceMonitor", "monitor.start", "time.perf_counter", "load_model_with_fallback", "tokenizer.encode", "sum", "range", "benchmark_single_prompt", "get_mlx_memory_info", "mlx_info.get", "enumerate", "results.append", "run_data.append", "monitor.sample", "BenchmarkSummary", "statistics.mean", "min", "max", "calculate_percentile", "monitor.get_summary" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "summary" ], "awaits": false, "yields": false }, { "name": "is_mllm_model", "qualname": "is_mllm_model", "full_name": "vllm_mlx.benchmark.is_mllm_model", "kind": "function", "signature": "def is_mllm_model(model_name: str) -> bool", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if model name indicates a multimodal language model.", "summary": "Check if model name indicates a multimodal language model.", "implementation": "Function `is_mllm_model` calls `model_name.lower`, `pattern.lower`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 651, "end_line": 657, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L651-L657", "decorators": [], "calls": [ "model_name.lower", "pattern.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "MLLMBenchmarkResult", "qualname": "MLLMBenchmarkResult", "full_name": "vllm_mlx.benchmark.MLLMBenchmarkResult", "kind": "class", "signature": "class MLLMBenchmarkResult", "parameters": [ { "name": "resolution", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "width", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "height", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "pixels", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "time_seconds", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens_generated", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens_per_second", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "response_preview", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "mlx_memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "MLLMBenchmarkResult", "docstring": "Result from a single MLLM benchmark run.", "summary": "Result from a single MLLM benchmark run.", "implementation": "Class `MLLMBenchmarkResult` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 661, "end_line": 674, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L661-L674", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "download_test_image", "qualname": "download_test_image", "full_name": "vllm_mlx.benchmark.download_test_image", "kind": "function", "signature": "def download_test_image(url: str, timeout: int=30) -> Image.Image", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "30", "required": false, "description": "Optional positional or keyword input; defaults to `30`." } ], "return_annotation": "Image.Image", "docstring": "Download image from URL and return PIL Image.", "summary": "Download image from URL and return PIL Image.", "implementation": "Function `download_test_image` calls `requests.get`, `response.raise_for_status`, `Image.open`, `io.BytesIO`; returns `Image.open(io.BytesIO(response.content))`.", "documented": true, "public": true, "addressable": true, "line": 677, "end_line": 684, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L677-L684", "decorators": [], "calls": [ "requests.get", "response.raise_for_status", "Image.open", "io.BytesIO" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Image.open(io.BytesIO(response.content))" ], "awaits": false, "yields": false }, { "name": "resize_image", "qualname": "resize_image", "full_name": "vllm_mlx.benchmark.resize_image", "kind": "function", "signature": "def resize_image(img: Image.Image, width: int, height: int) -> Image.Image", "parameters": [ { "name": "img", "kind": "positional or keyword", "annotation": "Image.Image", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "width", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "height", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Image.Image", "docstring": "Resize image to specified dimensions.", "summary": "Resize image to specified dimensions.", "implementation": "Function `resize_image` calls `img.resize`; returns `img.resize((width, height), Image.Resampling.LANCZOS)`.", "documented": true, "public": true, "addressable": true, "line": 687, "end_line": 689, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L687-L689", "decorators": [], "calls": [ "img.resize" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "img.resize((width, height), Image.Resampling.LANCZOS)" ], "awaits": false, "yields": false }, { "name": "image_to_base64", "qualname": "image_to_base64", "full_name": "vllm_mlx.benchmark.image_to_base64", "kind": "function", "signature": "def image_to_base64(img: Image.Image, format: str='JPEG') -> str", "parameters": [ { "name": "img", "kind": "positional or keyword", "annotation": "Image.Image", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "format", "kind": "positional or keyword", "annotation": "str", "default": "'JPEG'", "required": false, "description": "Optional positional or keyword input; defaults to `'JPEG'`." } ], "return_annotation": "str", "docstring": "Convert PIL Image to base64 data URL.", "summary": "Convert PIL Image to base64 data URL.", "implementation": "Function `image_to_base64` calls `Image.new`, `background.paste`, `img.split`, `img.convert`; returns `f'data:{mime};base64,{b64}'`.", "documented": true, "public": true, "addressable": true, "line": 692, "end_line": 705, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L692-L705", "decorators": [], "calls": [ "Image.new", "background.paste", "img.split", "img.convert", "io.BytesIO", "img.save", "base64.b64encode(buffer.getvalue()).decode", "base64.b64encode", "buffer.getvalue" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'data:{mime};base64,{b64}'" ], "awaits": false, "yields": false }, { "name": "benchmark_mllm_resolution", "qualname": "benchmark_mllm_resolution", "full_name": "vllm_mlx.benchmark.benchmark_mllm_resolution", "kind": "function", "signature": "def benchmark_mllm_resolution(model, processor, config, base_image: Image.Image, width: int, height: int, max_tokens: int=256, warmup: bool=False) -> MLLMBenchmarkResult", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "processor", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "base_image", "kind": "positional or keyword", "annotation": "Image.Image", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "width", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "height", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Optional positional or keyword input; defaults to `256`." }, { "name": "warmup", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "MLLMBenchmarkResult", "docstring": "Run MLLM benchmark for a specific resolution.", "summary": "Run MLLM benchmark for a specific resolution.", "implementation": "Function `benchmark_mllm_resolution` calls `reset_mlx_peak_memory`, `resize_image`, `tempfile.NamedTemporaryFile`, `img.save`; returns `MLLMBenchmarkResult(resolution=resolution_name, width=width, height=height, pixels=pixels, time_seconds=elapsed, tokens…`.", "documented": true, "public": true, "addressable": true, "line": 708, "end_line": 800, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L708-L800", "decorators": [], "calls": [ "reset_mlx_peak_memory", "resize_image", "tempfile.NamedTemporaryFile", "img.save", "print", "apply_chat_template", "time.perf_counter", "generate", "hasattr", "getattr", "len", "text.split", "str", "get_mlx_memory_info", "get_process_memory", "mlx_info.get", "os.unlink", "MLLMBenchmarkResult" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "MLLMBenchmarkResult(resolution=resolution_name, width=width, height=height, pixels=pixels, time_seconds=elapsed, tokens…" ], "awaits": false, "yields": false }, { "name": "run_mllm_benchmark", "qualname": "run_mllm_benchmark", "full_name": "vllm_mlx.benchmark.run_mllm_benchmark", "kind": "function", "signature": "def run_mllm_benchmark(model_name: str, quick: bool=False, max_tokens: int=256, warmup_runs: int=1) -> list[MLLMBenchmarkResult]", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name" }, { "name": "quick", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "If True, test only 4 resolutions" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Max tokens to generate" }, { "name": "warmup_runs", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Number of warmup runs" } ], "return_annotation": "list[MLLMBenchmarkResult]", "docstring": "Run MLLM benchmark across multiple image resolutions.\n\nArgs:\n model_name: HuggingFace model name\n quick: If True, test only 4 resolutions\n max_tokens: Max tokens to generate\n warmup_runs: Number of warmup runs\n\nReturns:\n List of MLLMBenchmarkResult", "summary": "Run MLLM benchmark across multiple image resolutions.", "implementation": "Function `run_mllm_benchmark` calls `detect_hardware`, `print`, `len`, `tabulate`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 803, "end_line": 913, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L803-L913", "decorators": [], "calls": [ "detect_hardware", "print", "len", "tabulate", "time.perf_counter", "load", "load_config", "download_test_image", "range", "benchmark_mllm_resolution", "get_mlx_memory_info", "mlx_info.get", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "results" ], "awaits": false, "yields": false }, { "name": "print_mllm_summary", "qualname": "print_mllm_summary", "full_name": "vllm_mlx.benchmark.print_mllm_summary", "kind": "function", "signature": "def print_mllm_summary(results: list[MLLMBenchmarkResult], model_name: str)", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[MLLMBenchmarkResult]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print MLLM benchmark summary.", "summary": "Print MLLM benchmark summary.", "implementation": "Function `print_mllm_summary` calls `print`, `table_data.append`, `tabulate`, `sum`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 916, "end_line": 975, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L916-L975", "decorators": [], "calls": [ "print", "table_data.append", "tabulate", "sum", "max", "min" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "VideoBenchmarkResult", "qualname": "VideoBenchmarkResult", "full_name": "vllm_mlx.benchmark.VideoBenchmarkResult", "kind": "class", "signature": "class VideoBenchmarkResult", "parameters": [ { "name": "config_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "fps", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "max_frames", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "frames_extracted", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "video_duration", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "time_seconds", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "completion_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens_per_second", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "response_preview", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "mlx_memory_gb", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "VideoBenchmarkResult", "docstring": "Result from a single video benchmark run.", "summary": "Result from a single video benchmark run.", "implementation": "Class `VideoBenchmarkResult` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 984, "end_line": 999, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L984-L999", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "create_test_video", "qualname": "create_test_video", "full_name": "vllm_mlx.benchmark.create_test_video", "kind": "function", "signature": "def create_test_video(duration: float=10.0, fps: float=30.0, width: int=640, height: int=480) -> str", "parameters": [ { "name": "duration", "kind": "positional or keyword", "annotation": "float", "default": "10.0", "required": false, "description": "Optional positional or keyword input; defaults to `10.0`." }, { "name": "fps", "kind": "positional or keyword", "annotation": "float", "default": "30.0", "required": false, "description": "Optional positional or keyword input; defaults to `30.0`." }, { "name": "width", "kind": "positional or keyword", "annotation": "int", "default": "640", "required": false, "description": "Optional positional or keyword input; defaults to `640`." }, { "name": "height", "kind": "positional or keyword", "annotation": "int", "default": "480", "required": false, "description": "Optional positional or keyword input; defaults to `480`." } ], "return_annotation": "str", "docstring": "Create a synthetic test video with colored frames and text.", "summary": "Create a synthetic test video with colored frames and text.", "implementation": "Function `create_test_video` calls `tempfile.NamedTemporaryFile`, `temp_file.close`, `cv2.VideoWriter_fourcc`, `cv2.VideoWriter`; returns `temp_file.name`.", "documented": true, "public": true, "addressable": true, "line": 1002, "end_line": 1056, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1002-L1056", "decorators": [], "calls": [ "tempfile.NamedTemporaryFile", "temp_file.close", "cv2.VideoWriter_fourcc", "cv2.VideoWriter", "int", "len", "range", "np.zeros", "min", "cv2.putText", "out.write", "out.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "temp_file.name" ], "awaits": false, "yields": false }, { "name": "download_video", "qualname": "download_video", "full_name": "vllm_mlx.benchmark.download_video", "kind": "function", "signature": "def download_video(url: str, timeout: int=120) -> str", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "120", "required": false, "description": "Optional positional or keyword input; defaults to `120`." } ], "return_annotation": "str", "docstring": "Download video from URL and return local path.", "summary": "Download video from URL and return local path.", "implementation": "Function `download_video` calls `print`, `requests.get`, `response.raise_for_status`, `tempfile.NamedTemporaryFile`; returns `temp_file.name`.", "documented": true, "public": true, "addressable": true, "line": 1059, "end_line": 1075, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1059-L1075", "decorators": [], "calls": [ "print", "requests.get", "response.raise_for_status", "tempfile.NamedTemporaryFile", "response.iter_content", "temp_file.write", "temp_file.close", "Path(temp_file.name).stat", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "temp_file.name" ], "awaits": false, "yields": false }, { "name": "get_video_info", "qualname": "get_video_info", "full_name": "vllm_mlx.benchmark.get_video_info", "kind": "function", "signature": "def get_video_info(video_path: str) -> dict", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Get information about a video file.", "summary": "Get information about a video file.", "implementation": "Function `get_video_info` calls `cv2.VideoCapture`, `cap.isOpened`, `int`, `cap.get`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1078, "end_line": 1094, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1078-L1094", "decorators": [], "calls": [ "cv2.VideoCapture", "cap.isOpened", "int", "cap.get", "cap.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'error': 'Cannot open video'}", "info" ], "awaits": false, "yields": false }, { "name": "benchmark_video_config", "qualname": "benchmark_video_config", "full_name": "vllm_mlx.benchmark.benchmark_video_config", "kind": "function", "signature": "def benchmark_video_config(model, video_path: str, fps: float, max_frames: int, config_name: str, video_info: dict, max_tokens: int=150, warmup: bool=False) -> VideoBenchmarkResult", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "fps", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_frames", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_info", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "150", "required": false, "description": "Optional positional or keyword input; defaults to `150`." }, { "name": "warmup", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "VideoBenchmarkResult", "docstring": "Run a single video benchmark configuration.", "summary": "Run a single video benchmark configuration.", "implementation": "Function `benchmark_video_config` calls `reset_mlx_peak_memory`, `print`, `time.perf_counter`, `model.generate`; returns `VideoBenchmarkResult(config_name=config_name, fps=fps, max_frames=max_frames, frames_extracted=frames_extracted, video_…`.", "documented": true, "public": true, "addressable": true, "line": 1097, "end_line": 1162, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1097-L1162", "decorators": [], "calls": [ "reset_mlx_peak_memory", "print", "time.perf_counter", "model.generate", "int", "min", "get_mlx_memory_info", "get_process_memory", "mlx_info.get", "VideoBenchmarkResult", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "VideoBenchmarkResult(config_name=config_name, fps=fps, max_frames=max_frames, frames_extracted=frames_extracted, video_…" ], "awaits": false, "yields": false }, { "name": "run_video_benchmark", "qualname": "run_video_benchmark", "full_name": "vllm_mlx.benchmark.run_video_benchmark", "kind": "function", "signature": "def run_video_benchmark(model_name: str, video_url: str=None, video_path: str=None, quick: bool=False, max_tokens: int=150, warmup_runs: int=1) -> list[VideoBenchmarkResult]", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace MLLM model name" }, { "name": "video_url", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "URL to download video from" }, { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Local video file path" }, { "name": "quick", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "If True, test only 3 configurations" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "150", "required": false, "description": "Max tokens to generate" }, { "name": "warmup_runs", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Number of warmup runs" } ], "return_annotation": "list[VideoBenchmarkResult]", "docstring": "Run video benchmark across multiple frame configurations.\n\nArgs:\n model_name: HuggingFace MLLM model name\n video_url: URL to download video from\n video_path: Local video file path\n quick: If True, test only 3 configurations\n max_tokens: Max tokens to generate\n warmup_runs: Number of warmup runs\n\nReturns:\n List of VideoBenchmarkResult", "summary": "Run video benchmark across multiple frame configurations.", "implementation": "Function `run_video_benchmark` calls `detect_hardware`, `print`, `len`, `tabulate`; returns `results`.", "documented": true, "public": true, "addressable": true, "line": 1165, "end_line": 1285, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1165-L1285", "decorators": [], "calls": [ "detect_hardware", "print", "len", "tabulate", "time.perf_counter", "MLXMultimodalLM", "model.load", "Path(video_path).exists", "Path", "download_video", "get_video_info", "range", "benchmark_video_config", "get_mlx_memory_info", "mlx_info.get", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "print_video_summary", "qualname": "print_video_summary", "full_name": "vllm_mlx.benchmark.print_video_summary", "kind": "function", "signature": "def print_video_summary(results: list[VideoBenchmarkResult], model_name: str)", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "list[VideoBenchmarkResult]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print video benchmark summary.", "summary": "Print video benchmark summary.", "implementation": "Function `print_video_summary` calls `print`, `sorted`, `table_data.append`, `tabulate`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 1288, "end_line": 1341, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1288-L1341", "decorators": [], "calls": [ "print", "sorted", "table_data.append", "tabulate", "sum", "max", "min" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "print_summary", "qualname": "print_summary", "full_name": "vllm_mlx.benchmark.print_summary", "kind": "function", "signature": "def print_summary(summary: BenchmarkSummary)", "parameters": [ { "name": "summary", "kind": "positional or keyword", "annotation": "BenchmarkSummary", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Print a formatted summary of benchmark results using tabulate.", "summary": "Print a formatted summary of benchmark results using tabulate.", "implementation": "Function `print_summary` calls `print`, `tabulate`, `resource_data.append`.", "documented": true, "public": true, "addressable": true, "line": 1349, "end_line": 1441, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1349-L1441", "decorators": [], "calls": [ "print", "tabulate", "resource_data.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "vllm_mlx.benchmark.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "Run the benchmark.", "summary": "Run the benchmark.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `is_mllm_model`.", "documented": true, "public": true, "addressable": true, "line": 1444, "end_line": 1680, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/benchmark.py#L1444-L1680", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "is_mllm_model", "run_video_benchmark", "print_video_summary", "open", "json.dump", "print", "run_mllm_benchmark", "print_mllm_summary", "run_benchmark", "print_summary" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.cli", "path": "vllm_mlx/cli.py", "page_path": "reference/api/vllm_mlx/cli.md", "docstring": "CLI for vllm-mlx.\n\nCommands:\n vllm-mlx serve --port 8000 Start OpenAI-compatible server\n vllm-mlx bench Run benchmark\n\nUsage:\n vllm-mlx serve mlx-community/Llama-3.2-3B-Instruct-4bit --port 8000\n vllm-mlx bench mlx-community/Llama-3.2-1B-Instruct-4bit --num-prompts 10", "summary": "CLI for vllm-mlx.", "line_count": 2138, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L1-L2138", "members": [ "serve_command", "download_command", "model_command", "bench_command", "bench_detok_command", "bench_kv_cache_command", "bench_serve_command", "create_parser", "build_parser", "main" ], "symbols": [ { "name": "serve_command", "qualname": "serve_command", "full_name": "vllm_mlx.cli.serve_command", "kind": "function", "signature": "def serve_command(args)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Start the OpenAI-compatible server.", "summary": "Start the OpenAI-compatible server.", "implementation": "Function `serve_command` calls `logging.getLogger`, `getattr`, `print`, `sys.exit`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 393, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L22-L393", "decorators": [], "calls": [ "logging.getLogger", "getattr", "print", "sys.exit", "server._metrics.configure", "RateLimiter", "get_parser", "parser_cls", "logger.info", "DownloadConfig", "ensure_model_downloaded", "is_mllm_model", "server.load_embedding_model", "server.load_reranker_model", "SchedulerConfig", "RegistryServeDefaults", "load_model_registry", "load_model", "uvicorn.run" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "download_command", "qualname": "download_command", "full_name": "vllm_mlx.cli.download_command", "kind": "function", "signature": "def download_command(args)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Download a model to local cache without starting a server.", "summary": "Download a model to local cache without starting a server.", "implementation": "Function `download_command` calls `DownloadConfig`, `print`, `ensure_model_downloaded`.", "documented": true, "public": true, "addressable": true, "line": 396, "end_line": 410, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L396-L410", "decorators": [], "calls": [ "DownloadConfig", "print", "ensure_model_downloaded" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "model_command", "qualname": "model_command", "full_name": "vllm_mlx.cli.model_command", "kind": "function", "signature": "def model_command(args)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Run model lifecycle helper commands.", "summary": "Run model lifecycle helper commands.", "implementation": "Function `model_command` calls `inspect_model`, `acquire_model`, `AcquisitionOptions`, `convert_model`; can raise `ValueError`.", "documented": true, "public": true, "addressable": true, "line": 413, "end_line": 503, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L413-L503", "decorators": [], "calls": [ "inspect_model", "acquire_model", "AcquisitionOptions", "convert_model", "ConversionOptions", "payload.get", "print", "json.dumps", "sys.exit", "register_model", "RegistrationOptions", "qualify_model", "QualificationOptions", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "bench_command", "qualname": "bench_command", "full_name": "vllm_mlx.cli.bench_command", "kind": "function", "signature": "def bench_command(args)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Run benchmark.", "summary": "Run benchmark.", "implementation": "Function `bench_command` calls `asyncio.run`, `run_benchmark`.", "documented": true, "public": true, "addressable": true, "line": 506, "end_line": 625, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L506-L625", "decorators": [], "calls": [ "asyncio.run", "run_benchmark" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "run_benchmark", "qualname": "bench_command.run_benchmark", "full_name": "vllm_mlx.cli.bench_command.run_benchmark", "kind": "nested function", "signature": "async def run_benchmark()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `bench_command.run_benchmark` calls `print`, `load`, `SchedulerConfig`, `EngineConfig`; awaits asynchronous work.", "implementation": "Nested Function `bench_command.run_benchmark` calls `print`, `load`, `SchedulerConfig`, `EngineConfig`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 520, "end_line": 623, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L520-L623", "decorators": [], "calls": [ "print", "load", "SchedulerConfig", "EngineConfig", "SamplingParams", "len", "AsyncEngineCore", "asyncio.sleep", "time.perf_counter", "engine.add_request", "request_ids.append", "asyncio.gather", "get_output" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "get_output", "qualname": "bench_command.run_benchmark.get_output", "full_name": "vllm_mlx.cli.bench_command.run_benchmark.get_output", "kind": "nested function", "signature": "async def get_output(rid)", "parameters": [ { "name": "rid", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `bench_command.run_benchmark.get_output` calls `engine.stream_outputs`; has 2 explicit return paths.", "implementation": "Nested Function `bench_command.run_benchmark.get_output` calls `engine.stream_outputs`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 597, "end_line": 601, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L597-L601", "decorators": [], "calls": [ "engine.stream_outputs" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "out", "None" ], "awaits": false, "yields": false }, { "name": "bench_detok_command", "qualname": "bench_detok_command", "full_name": "vllm_mlx.cli.bench_detok_command", "kind": "function", "signature": "def bench_detok_command(args)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Benchmark streaming detokenizer optimization.", "summary": "Benchmark streaming detokenizer optimization.", "implementation": "Function `bench_detok_command` calls `print`, `load`, `generate`, `tokenizer.encode`.", "documented": true, "public": true, "addressable": true, "line": 628, "end_line": 740, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L628-L740", "decorators": [], "calls": [ "print", "load", "generate", "tokenizer.encode", "len", "range", "time.perf_counter", "tokenizer.decode", "naive_times.append", "statistics.mean", "detok_class", "detok.reset", "detok.add_token", "detok.finalize", "streaming_times.append", "detok.text.strip", "batch_result.strip", "min", "repr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "bench_kv_cache_command", "qualname": "bench_kv_cache_command", "full_name": "vllm_mlx.cli.bench_kv_cache_command", "kind": "function", "signature": "def bench_kv_cache_command(args)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Benchmark KV cache quantization memory savings and quality.", "summary": "Benchmark KV cache quantization memory savings and quality.", "implementation": "Function `bench_kv_cache_command` calls `print`, `range`, `KVCache`, `mx.random.normal`.", "documented": true, "public": true, "addressable": true, "line": 743, "end_line": 886, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L743-L886", "decorators": [], "calls": [ "print", "range", "KVCache", "mx.random.normal", "cache.append", "mx.eval", "estimate_kv_cache_memory", "time.perf_counter", "_quantize_cache", "hasattr", "_dequantize_cache", "zip", "mx.abs(orig.keys - rest.keys).mean().item", "mx.abs(orig.keys - rest.keys).mean", "mx.abs", "mx.abs(orig.values - rest.values).mean().item", "mx.abs(orig.values - rest.values).mean", "mx.abs(orig.keys - rest.keys).max().item", "mx.abs(orig.keys - rest.keys).max", "mx.abs(orig.values - rest.values).max().item", "mx.abs(orig.values - rest.values).max", "max", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "bench_serve_command", "qualname": "bench_serve_command", "full_name": "vllm_mlx.cli.bench_serve_command", "kind": "function", "signature": "def bench_serve_command(args)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Run serving benchmark.", "summary": "Run serving benchmark.", "implementation": "Function `bench_serve_command` calls `sweep_only_warnings.append`, `print`, `', '.join`, `asyncio.run`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 889, "end_line": 990, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L889-L990", "decorators": [], "calls": [ "sweep_only_warnings.append", "print", "', '.join", "asyncio.run", "run_bench_serve_workload", "args.prompts.split", "int", "args.concurrency.split", "args.enable_thinking.split", "v.strip().lower", "v.strip", "thinking_values.append", "s.strip().strip", "s.strip", "re.split", "kv.split", "run_bench_serve", "bool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "create_parser", "qualname": "create_parser", "full_name": "vllm_mlx.cli.create_parser", "kind": "function", "signature": "def create_parser() -> argparse.ArgumentParser", "parameters": [], "return_annotation": "argparse.ArgumentParser", "docstring": "Build the top-level CLI parser.", "summary": "Build the top-level CLI parser.", "implementation": "Function `create_parser` calls `argparse.ArgumentParser`, `parser.add_subparsers`, `subparsers.add_parser`, `serve_parser.add_argument`; returns `parser`.", "documented": true, "public": true, "addressable": true, "line": 993, "end_line": 2105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L993-L2105", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_subparsers", "subparsers.add_parser", "serve_parser.add_argument", "make_positive_int_arg_parser", "list_parsers", "', '.join", "make_json_object_arg_parser", "bench_parser.add_argument", "detok_parser.add_argument", "kv_cache_parser.add_argument", "download_parser.add_argument", "model_parser.add_subparsers", "model_subparsers.add_parser", "model_inspect_parser.add_argument", "model_acquire_parser.add_argument", "model_convert_parser.add_argument", "model_register_parser.add_argument", "model_register_parser.add_mutually_exclusive_group", "mllm_group.add_argument", "model_qualify_parser.add_argument", "bench_serve_parser.add_argument" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parser" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "vllm_mlx.cli.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "Parse the command line and dispatch to the selected vllm-mlx command.", "summary": "Parse the command line and dispatch to the selected vllm-mlx command.", "implementation": "Function `main` calls `create_parser`, `parser.parse_args`, `serve_command`, `bench_command`.", "documented": true, "public": true, "addressable": true, "line": 2112, "end_line": 2134, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli.py#L2112-L2134", "decorators": [], "calls": [ "create_parser", "parser.parse_args", "serve_command", "bench_command", "bench_detok_command", "bench_kv_cache_command", "download_command", "model_command", "bench_serve_command", "parser.print_help", "sys.exit" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.cli_arg_types", "path": "vllm_mlx/cli_arg_types.py", "page_path": "reference/api/vllm_mlx/cli_arg_types.md", "docstring": "Argparse type helpers shared by CLI entrypoints.", "summary": "Argparse type helpers shared by CLI entrypoints.", "line_count": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L1-L62", "members": [ "parse_json_object_arg", "make_json_object_arg_parser", "positive_int_arg", "parse_positive_int_arg", "make_positive_int_arg_parser" ], "symbols": [ { "name": "parse_json_object_arg", "qualname": "parse_json_object_arg", "full_name": "vllm_mlx.cli_arg_types.parse_json_object_arg", "kind": "function", "signature": "def parse_json_object_arg(value: str, option_name: str) -> dict[str, Any]", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "option_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Parse and validate that an option value is a JSON object.", "summary": "Parse and validate that an option value is a JSON object.", "implementation": "Function `parse_json_object_arg` calls `json.loads`, `argparse.ArgumentTypeError`, `isinstance`; can raise `argparse.ArgumentTypeError`; returns `parsed`.", "documented": true, "public": true, "addressable": true, "line": 10, "end_line": 22, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L10-L22", "decorators": [], "calls": [ "json.loads", "argparse.ArgumentTypeError", "isinstance" ], "state_reads": [], "state_writes": [], "raises": [ "argparse.ArgumentTypeError" ], "return_expressions": [ "parsed" ], "awaits": false, "yields": false }, { "name": "make_json_object_arg_parser", "qualname": "make_json_object_arg_parser", "full_name": "vllm_mlx.cli_arg_types.make_json_object_arg_parser", "kind": "function", "signature": "def make_json_object_arg_parser(option_name: str) -> Callable[[str], dict[str, Any]]", "parameters": [ { "name": "option_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Callable[[str], dict[str, Any]]", "docstring": "Create an argparse type parser for JSON object options.", "summary": "Create an argparse type parser for JSON object options.", "implementation": "Function `make_json_object_arg_parser` returns `_parser`.", "documented": true, "public": true, "addressable": true, "line": 25, "end_line": 31, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L25-L31", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_parser" ], "awaits": false, "yields": false }, { "name": "_parser", "qualname": "make_json_object_arg_parser._parser", "full_name": "vllm_mlx.cli_arg_types.make_json_object_arg_parser._parser", "kind": "nested function", "signature": "def _parser(value: str) -> dict[str, Any]", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Nested Function `make_json_object_arg_parser._parser` calls `parse_json_object_arg`; returns `parse_json_object_arg(value, option_name)`.", "implementation": "Nested Function `make_json_object_arg_parser._parser` calls `parse_json_object_arg`; returns `parse_json_object_arg(value, option_name)`.", "documented": false, "public": false, "addressable": false, "line": 28, "end_line": 29, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L28-L29", "decorators": [], "calls": [ "parse_json_object_arg" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parse_json_object_arg(value, option_name)" ], "awaits": false, "yields": false }, { "name": "positive_int_arg", "qualname": "positive_int_arg", "full_name": "vllm_mlx.cli_arg_types.positive_int_arg", "kind": "function", "signature": "def positive_int_arg(value: str) -> int", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Parse an argparse integer that must be greater than zero.", "summary": "Parse an argparse integer that must be greater than zero.", "implementation": "Function `positive_int_arg` calls `int`, `argparse.ArgumentTypeError`; can raise `argparse.ArgumentTypeError`; returns `parsed`.", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 42, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L34-L42", "decorators": [], "calls": [ "int", "argparse.ArgumentTypeError" ], "state_reads": [], "state_writes": [], "raises": [ "argparse.ArgumentTypeError" ], "return_expressions": [ "parsed" ], "awaits": false, "yields": false }, { "name": "parse_positive_int_arg", "qualname": "parse_positive_int_arg", "full_name": "vllm_mlx.cli_arg_types.parse_positive_int_arg", "kind": "function", "signature": "def parse_positive_int_arg(value: str, option_name: str) -> int", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "option_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Parse and validate that an option value is a positive integer.", "summary": "Parse and validate that an option value is a positive integer.", "implementation": "Function `parse_positive_int_arg` calls `int`, `argparse.ArgumentTypeError`; can raise `argparse.ArgumentTypeError`; returns `parsed`.", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 53, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L45-L53", "decorators": [], "calls": [ "int", "argparse.ArgumentTypeError" ], "state_reads": [], "state_writes": [], "raises": [ "argparse.ArgumentTypeError" ], "return_expressions": [ "parsed" ], "awaits": false, "yields": false }, { "name": "make_positive_int_arg_parser", "qualname": "make_positive_int_arg_parser", "full_name": "vllm_mlx.cli_arg_types.make_positive_int_arg_parser", "kind": "function", "signature": "def make_positive_int_arg_parser(option_name: str) -> Callable[[str], int]", "parameters": [ { "name": "option_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Callable[[str], int]", "docstring": "Create an argparse type parser for positive integer options.", "summary": "Create an argparse type parser for positive integer options.", "implementation": "Function `make_positive_int_arg_parser` returns `_parser`.", "documented": true, "public": true, "addressable": true, "line": 56, "end_line": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L56-L62", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_parser" ], "awaits": false, "yields": false }, { "name": "_parser", "qualname": "make_positive_int_arg_parser._parser", "full_name": "vllm_mlx.cli_arg_types.make_positive_int_arg_parser._parser", "kind": "nested function", "signature": "def _parser(value: str) -> int", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "", "summary": "Nested Function `make_positive_int_arg_parser._parser` calls `parse_positive_int_arg`; returns `parse_positive_int_arg(value, option_name)`.", "implementation": "Nested Function `make_positive_int_arg_parser._parser` calls `parse_positive_int_arg`; returns `parse_positive_int_arg(value, option_name)`.", "documented": false, "public": false, "addressable": false, "line": 59, "end_line": 60, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/cli_arg_types.py#L59-L60", "decorators": [], "calls": [ "parse_positive_int_arg" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parse_positive_int_arg(value, option_name)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.constrained", "path": "vllm_mlx/constrained/__init__.py", "page_path": "reference/api/vllm_mlx/constrained/index.md", "docstring": "Constrained decoding for grammar-guided generation.\n\nProvides logits processors that mask token probabilities during generation\nso the model can only emit sequences matching a target grammar (e.g. a JSON\nschema). Used by the ``response_format`` parameter on the chat completion\nand Anthropic Messages endpoints.", "summary": "Constrained decoding for grammar-guided generation.", "line_count": 23, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/__init__.py#L1-L23", "members": [ "__all__" ], "symbols": [] }, { "name": "vllm_mlx.constrained.cache", "path": "vllm_mlx/constrained/cache.py", "page_path": "reference/api/vllm_mlx/constrained/cache.md", "docstring": "Cache of ``TokenEnforcerTokenizerData`` objects keyed by tokenizer identity.\n\nBuilding ``TokenEnforcerTokenizerData`` requires iterating over the entire\nvocabulary (up to 200k tokens on MiniMax/GLM) and decoding each token. The\ncost is ~1-2 seconds per model and the result is independent of the JSON\nschema, so we cache it for the lifetime of the process.", "summary": "Cache of ``TokenEnforcerTokenizerData`` objects keyed by tokenizer identity.", "line_count": 186, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L1-L186", "members": [ "logger", "_CACHE", "_CACHE_LOCK", "_resolve_inner_tokenizer", "_build_regular_tokens_list", "_get_eos_token_id", "_get_vocab_size", "_decode_function", "get_tokenizer_data", "clear_cache" ], "symbols": [ { "name": "_resolve_inner_tokenizer", "qualname": "_resolve_inner_tokenizer", "full_name": "vllm_mlx.constrained.cache._resolve_inner_tokenizer", "kind": "function", "signature": "def _resolve_inner_tokenizer(tokenizer: Any) -> Any", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "VLM processors wrap the actual tokenizer under ``processor.tokenizer``.\n``mlx_lm.tokenizer_utils.TokenizerWrapper`` exposes it via ``_tokenizer``.\nReturn the most-unwrapped tokenizer that still has the HF\n``all_special_ids`` / ``eos_token_id`` surface.\n\nNote: on HF ``PreTrainedTokenizerFast``, ``_tokenizer`` points at the\nrust-level object which lacks ``all_special_ids``; unwrapping to that\nlevel would cause every special token (````, ````, ``\\n``,\n``<|think|>`` …) to leak into ``regular_tokens`` and end up in\n``TokenizerPrefixTree.root`` as an always-allowed token. We only\nunwrap when the inner layer still exposes ``all_special_ids``.", "summary": "VLM processors wrap the actual tokenizer under ``processor.tokenizer``.", "implementation": "Function `_resolve_inner_tokenizer` calls `getattr`, `hasattr`; returns `tokenizer`.", "documented": true, "public": false, "addressable": true, "line": 26, "end_line": 53, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L26-L53", "decorators": [], "calls": [ "getattr", "hasattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tokenizer" ], "awaits": false, "yields": false }, { "name": "_build_regular_tokens_list", "qualname": "_build_regular_tokens_list", "full_name": "vllm_mlx.constrained.cache._build_regular_tokens_list", "kind": "function", "signature": "def _build_regular_tokens_list(tokenizer: Any, vocab_size: int) -> list[tuple[int, str, bool]]", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "vocab_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[tuple[int, str, bool]]", "docstring": "Enumerate the regular (non-special) tokens in the vocabulary and produce\nthe ``(token_id, decoded_with_leading_space_marker, is_word_start)`` tuples\nrequired by ``TokenEnforcerTokenizerData``.\n\nMirrors the reference implementation in ``lmformatenforcer.integrations.\ntransformers`` but works with the HF tokenizer surface only (so we do not\nneed a hard transformers dependency at the right version).", "summary": "Enumerate the regular (non-special) tokens in the vocabulary and produce the ``(token_id, decoded_with_leading_space_marker, is_word_start)`` tuples required by ``TokenEnforcerTokenizerData``.", "implementation": "Function `_build_regular_tokens_list` calls `set`, `tokenizer.encode`, `range`, `tokenizer.decode`; returns `regular_tokens`.", "documented": true, "public": false, "addressable": true, "line": 56, "end_line": 95, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L56-L95", "decorators": [], "calls": [ "set", "tokenizer.encode", "range", "tokenizer.decode", "len", "regular_tokens.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "regular_tokens" ], "awaits": false, "yields": false }, { "name": "_get_eos_token_id", "qualname": "_get_eos_token_id", "full_name": "vllm_mlx.constrained.cache._get_eos_token_id", "kind": "function", "signature": "def _get_eos_token_id(tokenizer: Any) -> int | list[int]", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int | list[int]", "docstring": "", "summary": "Function `_get_eos_token_id` calls `getattr`, `isinstance`, `list`; has 3 explicit return paths.", "implementation": "Function `_get_eos_token_id` calls `getattr`, `isinstance`, `list`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 98, "end_line": 111, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L98-L111", "decorators": [], "calls": [ "getattr", "isinstance", "list" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "list(eos_list)", "eos", "0" ], "awaits": false, "yields": false }, { "name": "_get_vocab_size", "qualname": "_get_vocab_size", "full_name": "vllm_mlx.constrained.cache._get_vocab_size", "kind": "function", "signature": "def _get_vocab_size(tokenizer: Any) -> int", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "", "summary": "Function `_get_vocab_size` calls `getattr`, `isinstance`, `len`, `callable`; can raise `ValueError`; has 3 explicit return paths.", "implementation": "Function `_get_vocab_size` calls `getattr`, `isinstance`, `len`, `callable`; can raise `ValueError`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 114, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L114-L125", "decorators": [], "calls": [ "getattr", "isinstance", "len", "callable", "get_vocab", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "vs", "len(tokenizer)", "len(get_vocab())" ], "awaits": false, "yields": false }, { "name": "_decode_function", "qualname": "_decode_function", "full_name": "vllm_mlx.constrained.cache._decode_function", "kind": "function", "signature": "def _decode_function(tokenizer: Any, tokens: list[int]) -> str", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Function `_decode_function` calls `tokenizer.decode`, `isinstance`, `decoded.rstrip`; has 2 explicit return paths.", "implementation": "Function `_decode_function` calls `tokenizer.decode`, `isinstance`, `decoded.rstrip`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 128, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L128-L133", "decorators": [], "calls": [ "tokenizer.decode", "isinstance", "decoded.rstrip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''", "decoded.rstrip('�') if isinstance(decoded, str) else ''" ], "awaits": false, "yields": false }, { "name": "get_tokenizer_data", "qualname": "get_tokenizer_data", "full_name": "vllm_mlx.constrained.cache.get_tokenizer_data", "kind": "function", "signature": "def get_tokenizer_data(tokenizer: Any) -> Any | None", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any | None", "docstring": "Return a cached ``TokenEnforcerTokenizerData`` for ``tokenizer``.\n\nReturns ``None`` if ``lm-format-enforcer`` is not installed or the\ntokenizer cannot be adapted.", "summary": "Return a cached ``TokenEnforcerTokenizerData`` for ``tokenizer``.", "implementation": "Function `get_tokenizer_data` calls `_resolve_inner_tokenizer`, `id`, `_CACHE.get`, `_get_vocab_size`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 136, "end_line": 180, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L136-L180", "decorators": [], "calls": [ "_resolve_inner_tokenizer", "id", "_CACHE.get", "_get_vocab_size", "logger.warning", "_build_regular_tokens_list", "functools.partial", "_get_eos_token_id", "TokenEnforcerTokenizerData" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "cached", "data" ], "awaits": false, "yields": false }, { "name": "clear_cache", "qualname": "clear_cache", "full_name": "vllm_mlx.constrained.cache.clear_cache", "kind": "function", "signature": "def clear_cache() -> None", "parameters": [], "return_annotation": "None", "docstring": "Drop the cache (mainly for tests).", "summary": "Drop the cache (mainly for tests).", "implementation": "Function `clear_cache` calls `_CACHE.clear`.", "documented": true, "public": true, "addressable": true, "line": 183, "end_line": 186, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/cache.py#L183-L186", "decorators": [], "calls": [ "_CACHE.clear" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.constrained.json_schema_processor", "path": "vllm_mlx/constrained/json_schema_processor.py", "page_path": "reference/api/vllm_mlx/constrained/json_schema_processor.md", "docstring": "``JSONSchemaLogitsProcessor`` — a ``mlx_lm``-compatible logits processor that\nmasks the vocabulary so the model can only emit tokens forming a valid JSON\nvalue (optionally matching a JSON schema).\n\nThe processor implements the signature expected by ``mlx_lm.generate.generate_step``\nand ``vllm_mlx``'s batched engine alike:\n\n processor(tokens: mx.array, logits: mx.array) -> mx.array\n\n``tokens`` contains the full sequence generated for this request so far\n(prompt + previously emitted tokens), and ``logits`` is the last-step logits\nrow.", "summary": "``JSONSchemaLogitsProcessor`` — a ``mlx_lm``-compatible logits processor that masks the vocabulary so the model can only emit tokens forming a valid JSON value (optionally matching a JSON schema).", "line_count": 924, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L1-L924", "members": [ "logger", "LMFormatEnforcerNotAvailableError", "_parser_cache", "_MAX_NONPROGRESS_WHITESPACE_CHARS", "_JSON_WHITESPACE", "_canonical_schema_key", "_get_or_build_parser", "is_available", "_GENERIC_JSON_SCHEMA", "_simplify_schema", "_force_no_additional_properties", "_inject_no_additional_props", "_collect_property_names", "_walk_properties", "_complete_json_eos_logits", "_eos_logits", "_eos_logits_or_original", "JSONSchemaLogitsProcessor" ], "symbols": [ { "name": "LMFormatEnforcerNotAvailableError", "qualname": "LMFormatEnforcerNotAvailableError", "full_name": "vllm_mlx.constrained.json_schema_processor.LMFormatEnforcerNotAvailableError", "kind": "class", "signature": "class LMFormatEnforcerNotAvailableError(RuntimeError)", "parameters": [], "return_annotation": "LMFormatEnforcerNotAvailableError", "docstring": "Raised when ``lm-format-enforcer`` is required but not installed.", "summary": "Raised when ``lm-format-enforcer`` is required but not installed.", "implementation": "Class `LMFormatEnforcerNotAvailableError` derives from `RuntimeError` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 33, "end_line": 34, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L33-L34", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_canonical_schema_key", "qualname": "_canonical_schema_key", "full_name": "vllm_mlx.constrained.json_schema_processor._canonical_schema_key", "kind": "function", "signature": "def _canonical_schema_key(schema: dict | None) -> str", "parameters": [ { "name": "schema", "kind": "positional or keyword", "annotation": "dict | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Function `_canonical_schema_key` calls `json.dumps(schema, sort_keys=True, separators=(',', ':')).encode`, `json.dumps`, `hashlib.sha256(blob).hexdigest`, `hashlib.sha256`; has 2 explicit return paths.", "implementation": "Function `_canonical_schema_key` calls `json.dumps(schema, sort_keys=True, separators=(',', ':')).encode`, `json.dumps`, `hashlib.sha256(blob).hexdigest`, `hashlib.sha256`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 50, "end_line": 54, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L50-L54", "decorators": [], "calls": [ "json.dumps(schema, sort_keys=True, separators=(',', ':')).encode", "json.dumps", "hashlib.sha256(blob).hexdigest", "hashlib.sha256" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'__none__'", "hashlib.sha256(blob).hexdigest()" ], "awaits": false, "yields": false }, { "name": "_get_or_build_parser", "qualname": "_get_or_build_parser", "full_name": "vllm_mlx.constrained.json_schema_processor._get_or_build_parser", "kind": "function", "signature": "def _get_or_build_parser(schema: dict | None) -> tuple[dict, Any]", "parameters": [ { "name": "schema", "kind": "positional or keyword", "annotation": "dict | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[dict, Any]", "docstring": "Return (parser_schema, JsonSchemaParser) for ``schema``, memoised.", "summary": "Return (parser_schema, JsonSchemaParser) for ``schema``, memoised.", "implementation": "Function `_get_or_build_parser` calls `_canonical_schema_key`, `_parser_cache.get`, `_simplify_schema`, `_force_no_additional_properties`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 57, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L57-L73", "decorators": [], "calls": [ "_canonical_schema_key", "_parser_cache.get", "_simplify_schema", "_force_no_additional_properties", "JsonSchemaParser" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cached", "(parser_schema, parser)" ], "awaits": false, "yields": false }, { "name": "is_available", "qualname": "is_available", "full_name": "vllm_mlx.constrained.json_schema_processor.is_available", "kind": "function", "signature": "def is_available() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``True`` iff ``lm-format-enforcer`` is importable.", "summary": "Return ``True`` iff ``lm-format-enforcer`` is importable.", "implementation": "Function `is_available` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 76, "end_line": 82, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L76-L82", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_simplify_schema", "qualname": "_simplify_schema", "full_name": "vllm_mlx.constrained.json_schema_processor._simplify_schema", "kind": "function", "signature": "def _simplify_schema(schema: dict) -> dict", "parameters": [ { "name": "schema", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Pre-process a JSON Schema for ``lm-format-enforcer`` compatibility.\n\n``lm-format-enforcer`` does not support ``$ref``, ``not``, ``type`` as an\narray, or recursive definitions. This function:\n\n1. Resolves ``$ref`` by inlining referenced definitions (with cycle\n detection so recursive definitions are truncated to ``{}``).\n2. Removes ``not`` sub-schemas (makes the schema more permissive).\n3. Strips metadata / serialisation-hint keywords that the enforcer does\n not understand: ``default``, ``examples``, ``title``, ``description``,\n ``$schema``, ``$id``.\n4. Converts ``type: [t1, t2, ...]`` to ``anyOf: [{type: t1}, ...]``.\n5. Cleans up empty ``anyOf`` / ``oneOf`` branches.\n6. Flattens nested ``anyOf``/``oneOf`` (e.g.\n ``anyOf: [{anyOf: [A, B]}, C]`` → ``anyOf: [A, B, C]``).", "summary": "Pre-process a JSON Schema for ``lm-format-enforcer`` compatibility.", "implementation": "Function `_simplify_schema` calls `copy.deepcopy`, `definitions.update`, `schema.pop`, `set`; returns `_resolve(schema)`.", "documented": true, "public": false, "addressable": true, "line": 97, "end_line": 207, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L97-L207", "decorators": [], "calls": [ "copy.deepcopy", "definitions.update", "schema.pop", "set", "_resolve" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_resolve(schema)" ], "awaits": false, "yields": false }, { "name": "_resolve", "qualname": "_simplify_schema._resolve", "full_name": "vllm_mlx.constrained.json_schema_processor._simplify_schema._resolve", "kind": "nested function", "signature": "def _resolve(node: Any, depth: int=0) -> Any", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "depth", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "Any", "docstring": "", "summary": "Nested Function `_simplify_schema._resolve` calls `isinstance`, `ref.split`, `len`, `resolving.add`; has 3 explicit return paths.", "implementation": "Nested Function `_simplify_schema._resolve` calls `isinstance`, `ref.split`, `len`, `resolving.add`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 121, "end_line": 205, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L121-L205", "decorators": [], "calls": [ "isinstance", "ref.split", "len", "resolving.add", "copy.deepcopy", "node.items", "_resolve", "resolving.discard", "node.pop", "node.get", "branches.append", "list", "flattened.extend", "flattened.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "node", "result", "{}" ], "awaits": false, "yields": false }, { "name": "_force_no_additional_properties", "qualname": "_force_no_additional_properties", "full_name": "vllm_mlx.constrained.json_schema_processor._force_no_additional_properties", "kind": "function", "signature": "def _force_no_additional_properties(schema: dict) -> dict", "parameters": [ { "name": "schema", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Return a deep copy of *schema* with ``additionalProperties: false``\ninjected into every object-type sub-schema that declares ``properties``.\n\n``lm-format-enforcer`` has a bug where multi-character tokens spanning\nJSON structural boundaries (e.g., a single token that decodes to ``\"\"``)\ncan produce empty or whitespace-only keys, causing ``KeyError`` crashes in\n``jsonschemaparser.py``. Setting ``additionalProperties: false`` tells the\nenforcer's trie traversal that only the declared property names are valid\nkeys, which significantly narrows the allowed tokens and prevents most of\nthese boundary-spanning issues.", "summary": "Return a deep copy of *schema* with ``additionalProperties: false`` injected into every object-type sub-schema that declares ``properties``.", "implementation": "Function `_force_no_additional_properties` calls `copy.deepcopy`, `_inject_no_additional_props`; returns `schema`.", "documented": true, "public": false, "addressable": true, "line": 210, "end_line": 224, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L210-L224", "decorators": [], "calls": [ "copy.deepcopy", "_inject_no_additional_props" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "schema" ], "awaits": false, "yields": false }, { "name": "_inject_no_additional_props", "qualname": "_inject_no_additional_props", "full_name": "vllm_mlx.constrained.json_schema_processor._inject_no_additional_props", "kind": "function", "signature": "def _inject_no_additional_props(node: Any) -> None", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Recursively inject ``additionalProperties: false`` into *node*.", "summary": "Recursively inject ``additionalProperties: false`` into *node*.", "implementation": "Function `_inject_no_additional_props` calls `isinstance`, `node.values`, `_inject_no_additional_props`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 227, "end_line": 238, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L227-L238", "decorators": [], "calls": [ "isinstance", "node.values", "_inject_no_additional_props" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_collect_property_names", "qualname": "_collect_property_names", "full_name": "vllm_mlx.constrained.json_schema_processor._collect_property_names", "kind": "function", "signature": "def _collect_property_names(schema: dict | None) -> set[str]", "parameters": [ { "name": "schema", "kind": "positional or keyword", "annotation": "dict | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "set[str]", "docstring": "Collect all property names declared anywhere in *schema*.", "summary": "Collect all property names declared anywhere in *schema*.", "implementation": "Function `_collect_property_names` calls `set`, `_walk_properties`; returns `names`.", "documented": true, "public": false, "addressable": true, "line": 241, "end_line": 247, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L241-L247", "decorators": [], "calls": [ "set", "_walk_properties" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "names" ], "awaits": false, "yields": false }, { "name": "_walk_properties", "qualname": "_walk_properties", "full_name": "vllm_mlx.constrained.json_schema_processor._walk_properties", "kind": "function", "signature": "def _walk_properties(node: Any, names: set[str]) -> None", "parameters": [ { "name": "node", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "names", "kind": "positional or keyword", "annotation": "set[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Function `_walk_properties` calls `isinstance`, `node.get`, `names.update`, `props.keys`; returns `None`.", "implementation": "Function `_walk_properties` calls `isinstance`, `node.get`, `names.update`, `props.keys`; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 250, "end_line": 264, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L250-L264", "decorators": [], "calls": [ "isinstance", "node.get", "names.update", "props.keys", "props.values", "_walk_properties" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_complete_json_eos_logits", "qualname": "_complete_json_eos_logits", "full_name": "vllm_mlx.constrained.json_schema_processor._complete_json_eos_logits", "kind": "function", "signature": "def _complete_json_eos_logits(eos_set: set[int], suffix: list[int], logits: mx.array, is_complete_json, build_allow_mask) -> mx.array | None", "parameters": [ { "name": "eos_set", "kind": "positional or keyword", "annotation": "set[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "is_complete_json", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "build_allow_mask", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array | None", "docstring": "", "summary": "Function `_complete_json_eos_logits` calls `is_complete_json`, `_eos_logits`; has 2 explicit return paths.", "implementation": "Function `_complete_json_eos_logits` calls `is_complete_json`, `_eos_logits`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 267, "end_line": 276, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L267-L276", "decorators": [], "calls": [ "is_complete_json", "_eos_logits" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "_eos_logits(eos_set, logits, build_allow_mask)" ], "awaits": false, "yields": false }, { "name": "_eos_logits", "qualname": "_eos_logits", "full_name": "vllm_mlx.constrained.json_schema_processor._eos_logits", "kind": "function", "signature": "def _eos_logits(eos_set: set[int], logits: mx.array, build_allow_mask) -> mx.array | None", "parameters": [ { "name": "eos_set", "kind": "positional or keyword", "annotation": "set[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "build_allow_mask", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array | None", "docstring": "", "summary": "Function `_eos_logits` calls `build_allow_mask`, `sorted`; has 2 explicit return paths.", "implementation": "Function `_eos_logits` calls `build_allow_mask`, `sorted`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 279, "end_line": 290, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L279-L290", "decorators": [], "calls": [ "build_allow_mask", "sorted" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "logits + mask" ], "awaits": false, "yields": false }, { "name": "_eos_logits_or_original", "qualname": "_eos_logits_or_original", "full_name": "vllm_mlx.constrained.json_schema_processor._eos_logits_or_original", "kind": "function", "signature": "def _eos_logits_or_original(eos_set: set[int], logits: mx.array, build_allow_mask) -> mx.array", "parameters": [ { "name": "eos_set", "kind": "positional or keyword", "annotation": "set[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "build_allow_mask", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "", "summary": "Function `_eos_logits_or_original` calls `_eos_logits`; returns `logits if masked is None else masked`.", "implementation": "Function `_eos_logits_or_original` calls `_eos_logits`; returns `logits if masked is None else masked`.", "documented": false, "public": false, "addressable": true, "line": 293, "end_line": 299, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L293-L299", "decorators": [], "calls": [ "_eos_logits" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "logits if masked is None else masked" ], "awaits": false, "yields": false }, { "name": "JSONSchemaLogitsProcessor", "qualname": "JSONSchemaLogitsProcessor", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor", "kind": "class", "signature": "class JSONSchemaLogitsProcessor", "parameters": [ { "name": "schema", "kind": "positional or keyword", "annotation": "dict | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "JSONSchemaLogitsProcessor", "docstring": "Logits processor that constrains generation to valid JSON.\n\nParameters\n----------\nschema:\n The JSON Schema the output must match. When ``None``, any valid JSON\n object/array is accepted (``json_object`` mode).\ntokenizer:\n The tokenizer used for generation. Its vocabulary is iterated once\n (via :mod:`vllm_mlx.constrained.cache`) and cached for subsequent\n requests.", "summary": "Logits processor that constrains generation to valid JSON.", "implementation": "Class `JSONSchemaLogitsProcessor` declares 15 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 302, "end_line": 924, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L302-L924", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "JSONSchemaLogitsProcessor.__init__", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor.__init__", "kind": "method", "signature": "def __init__(self, schema: dict | None, tokenizer: Any) -> None", "parameters": [ { "name": "schema", "kind": "positional or keyword", "annotation": "dict | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `JSONSchemaLogitsProcessor.__init__` updates `self._tokenizer`, `self._schema`, `self._tok_data`, `self._disabled`; calls `is_available`, `LMFormatEnforcerNotAvailableError`, `get_tokenizer_data`, `_get_or_build_parser`; can raise `LMFormatEnforcerNotAvailableError`.", "implementation": "Method `JSONSchemaLogitsProcessor.__init__` updates `self._tokenizer`, `self._schema`, `self._tok_data`, `self._disabled`; calls `is_available`, `LMFormatEnforcerNotAvailableError`, `get_tokenizer_data`, `_get_or_build_parser`; can raise `LMFormatEnforcerNotAvailableError`.", "documented": false, "public": false, "addressable": true, "line": 317, "end_line": 414, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L317-L414", "decorators": [], "calls": [ "is_available", "LMFormatEnforcerNotAvailableError", "get_tokenizer_data", "_get_or_build_parser", "TokenEnforcer", "logger.warning", "self._enforcer.get_allowed_tokens", "_get_vocab_size", "getattr", "isinstance", "int", "set", "_collect_property_names" ], "state_reads": [ "self._tok_data", "self._parser", "self._disabled", "self._enforcer.get_allowed_tokens", "self._enforcer" ], "state_writes": [ "self._tokenizer", "self._schema", "self._tok_data", "self._disabled", "self._parser", "self._enforcer", "self._prompt_len", "self._vocab_size", "self._eos_set", "self._valid_key_first_chars", "self._valid_key_names", "self._token_decode_cache", "self._cached_suffix_text", "self._cached_suffix_len", "self._json_ctx_in_string", "self._json_ctx_last_quote_pos", "self._json_ctx_scanned_len", "self._brace_depth", "self._bracket_depth", "self._container_stack" ], "raises": [ "LMFormatEnforcerNotAvailableError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_suffix", "qualname": "JSONSchemaLogitsProcessor._suffix", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._suffix", "kind": "method", "signature": "def _suffix(self, tokens_list: list[int]) -> list[int]", "parameters": [ { "name": "tokens_list", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[int]", "docstring": "Return the slice of ``tokens`` that corresponds to generated output.", "summary": "Return the slice of ``tokens`` that corresponds to generated output.", "implementation": "Method `JSONSchemaLogitsProcessor._suffix` updates `self._prompt_len`; calls `len`; returns `tokens_list[self._prompt_len:]`.", "documented": true, "public": false, "addressable": true, "line": 418, "end_line": 426, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L418-L426", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._prompt_len" ], "state_writes": [ "self._prompt_len" ], "raises": [], "return_expressions": [ "tokens_list[self._prompt_len:]" ], "awaits": false, "yields": false }, { "name": "_decode_token_cached", "qualname": "JSONSchemaLogitsProcessor._decode_token_cached", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._decode_token_cached", "kind": "method", "signature": "def _decode_token_cached(self, tok_id: int) -> str | None", "parameters": [ { "name": "tok_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "Return the decoded text for a single token (cached).", "summary": "Return the decoded text for a single token (cached).", "implementation": "Method `JSONSchemaLogitsProcessor._decode_token_cached` calls `self._token_decode_cache.get`, `self._tokenizer.decode`, `isinstance`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 428, "end_line": 442, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L428-L442", "decorators": [], "calls": [ "self._token_decode_cache.get", "self._tokenizer.decode", "isinstance" ], "state_reads": [ "self._token_decode_cache.get", "self._token_decode_cache", "self._tokenizer.decode", "self._tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "cached", "None", "result" ], "awaits": false, "yields": false }, { "name": "_decode_suffix", "qualname": "JSONSchemaLogitsProcessor._decode_suffix", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._decode_suffix", "kind": "method", "signature": "def _decode_suffix(self, suffix: list[int]) -> str | None", "parameters": [ { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "Decode suffix tokens to text.\n\nAlways uses full ``tokenizer.decode(suffix)`` which is correct for\nall tokenizer families (BPE, SentencePiece, etc.). Per-token\nconcatenation is NOT safe because whitespace may be encoded as a\ntoken prefix (e.g. ``decode([1526]) = \"world\"`` but in context\n``decode([22557, 1526]) = \"Hello world\"``).\n\nResults are cached by suffix length to avoid redundant decodes\nwithin the same generation step (``_get_json_context`` and\n``_suffix_is_complete_json`` both call this method).", "summary": "Decode suffix tokens to text.", "implementation": "Method `JSONSchemaLogitsProcessor._decode_suffix` updates `self._cached_suffix_text`, `self._cached_suffix_len`, `self._json_ctx_scanned_len`, `self._json_ctx_in_string`; calls `len`, `self._tokenizer.decode`, `list`, `isinstance`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 444, "end_line": 493, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L444-L493", "decorators": [], "calls": [ "len", "self._tokenizer.decode", "list", "isinstance", "result.startswith" ], "state_reads": [ "self._cached_suffix_len", "self._cached_suffix_text", "self._tokenizer.decode", "self._tokenizer" ], "state_writes": [ "self._cached_suffix_text", "self._cached_suffix_len", "self._json_ctx_scanned_len", "self._json_ctx_in_string", "self._json_ctx_last_quote_pos", "self._brace_depth", "self._bracket_depth", "self._container_stack" ], "raises": [], "return_expressions": [ "''", "self._cached_suffix_text", "None", "result" ], "awaits": false, "yields": false }, { "name": "_suffix_is_complete_json", "qualname": "JSONSchemaLogitsProcessor._suffix_is_complete_json", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._suffix_is_complete_json", "kind": "method", "signature": "def _suffix_is_complete_json(self, suffix: list[int]) -> bool", "parameters": [ { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True if the decoded ``suffix`` parses as a complete JSON value.\n\nUses cached bracket/brace depth from ``_get_json_context`` as a\nfast pre-check: JSON cannot be complete when brackets are\nunbalanced or we are inside a string. This avoids the expensive\n``json.loads`` call on ~99% of steps.", "summary": "Return True if the decoded ``suffix`` parses as a complete JSON value.", "implementation": "Method `JSONSchemaLogitsProcessor._suffix_is_complete_json` calls `self._decode_suffix`, `text.strip`, `json.loads`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 495, "end_line": 520, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L495-L520", "decorators": [], "calls": [ "self._decode_suffix", "text.strip", "json.loads" ], "state_reads": [ "self._brace_depth", "self._bracket_depth", "self._json_ctx_in_string", "self._decode_suffix" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_get_json_context", "qualname": "JSONSchemaLogitsProcessor._get_json_context", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._get_json_context", "kind": "method", "signature": "def _get_json_context(self, suffix: list[int]) -> str", "parameters": [ { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Determine the JSON structural context of the current suffix.\n\nProcesses only newly appended characters instead of re-scanning\nthe full decoded text on every call (O(1) amortised per step\ninstead of O(n)).\n\nReturns one of:\n- ``\"key_start\"``: expecting a new key (after ``{`` or ``,``)\n- ``\"in_key\"``: inside an open key string\n- ``\"other\"``: any other position", "summary": "Determine the JSON structural context of the current suffix.", "implementation": "Method `JSONSchemaLogitsProcessor._get_json_context` updates `self._json_ctx_in_string`, `self._json_ctx_last_quote_pos`, `self._json_ctx_scanned_len`, `self._brace_depth`; calls `self._decode_suffix`, `len`, `container_stack.append`, `container_stack.pop`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 522, "end_line": 643, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L522-L643", "decorators": [], "calls": [ "self._decode_suffix", "len", "container_stack.append", "container_stack.pop", "text[:self._json_ctx_last_quote_pos].rstrip", "text.rstrip" ], "state_reads": [ "self._decode_suffix", "self._json_ctx_scanned_len", "self._json_ctx_in_string", "self._json_ctx_last_quote_pos", "self._brace_depth", "self._bracket_depth", "self._container_stack" ], "state_writes": [ "self._json_ctx_in_string", "self._json_ctx_last_quote_pos", "self._json_ctx_scanned_len", "self._brace_depth", "self._bracket_depth", "self._container_stack" ], "raises": [], "return_expressions": [ "'other'", "'in_key'", "'key_start'" ], "awaits": false, "yields": false }, { "name": "_filter_at_key_context", "qualname": "JSONSchemaLogitsProcessor._filter_at_key_context", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._filter_at_key_context", "kind": "method", "signature": "def _filter_at_key_context(self, context: str, suffix: list[int], allowed: list[int]) -> list[int]", "parameters": [ { "name": "context", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "allowed", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[int]", "docstring": "Apply schema-aware filtering when in key-related context.\n\nAt ``key_start``: only allow tokens that begin a valid key, whitespace,\n``}``, or just ``\"``.\nAt ``in_key``: only allow tokens compatible with continuing a valid\nproperty name (no leading whitespace; content must be a valid prefix).", "summary": "Apply schema-aware filtering when in key-related context.", "implementation": "Method `JSONSchemaLogitsProcessor._filter_at_key_context` calls `self._filter_key_start_tokens`, `self._filter_in_key_tokens`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 645, "end_line": 662, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L645-L662", "decorators": [], "calls": [ "self._filter_key_start_tokens", "self._filter_in_key_tokens" ], "state_reads": [ "self._valid_key_names", "self._filter_key_start_tokens", "self._filter_in_key_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "allowed", "self._filter_key_start_tokens(suffix, allowed)", "self._filter_in_key_tokens(suffix, allowed)" ], "awaits": false, "yields": false }, { "name": "_filter_key_start_tokens", "qualname": "JSONSchemaLogitsProcessor._filter_key_start_tokens", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._filter_key_start_tokens", "kind": "method", "signature": "def _filter_key_start_tokens(self, suffix: list[int], allowed: list[int]) -> list[int]", "parameters": [ { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "allowed", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[int]", "docstring": "Filter tokens at key-start position.\n\nOnly permit tokens that:\n- Are whitespace-only (before the key ``\"``)\n- Decode to ``}`` (close object)\n- Start a valid key: ``\"`` followed by a valid first char", "summary": "Filter tokens at key-start position.", "implementation": "Method `JSONSchemaLogitsProcessor._filter_key_start_tokens` calls `self._decode_token_cached`, `result.append`, `tok_text.lstrip`, `rest.find`; returns `result if result else allowed`.", "documented": true, "public": false, "addressable": true, "line": 664, "end_line": 717, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L664-L717", "decorators": [], "calls": [ "self._decode_token_cached", "result.append", "tok_text.lstrip", "rest.find", "self._is_valid_key_prefix" ], "state_reads": [ "self._eos_set", "self._decode_token_cached", "self._valid_key_first_chars", "self._is_valid_key_prefix", "self._valid_key_names" ], "state_writes": [], "raises": [], "return_expressions": [ "result if result else allowed" ], "awaits": false, "yields": false }, { "name": "_filter_in_key_tokens", "qualname": "JSONSchemaLogitsProcessor._filter_in_key_tokens", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._filter_in_key_tokens", "kind": "method", "signature": "def _filter_in_key_tokens(self, suffix: list[int], allowed: list[int]) -> list[int]", "parameters": [ { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "allowed", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[int]", "docstring": "Filter tokens when we're inside an open key string.\n\nOnly allow tokens whose content continues a valid property name.\nReject whitespace-only/leading-whitespace tokens.", "summary": "Filter tokens when we're inside an open key string.", "implementation": "Method `JSONSchemaLogitsProcessor._filter_in_key_tokens` calls `self._decode_suffix`, `text.rfind`, `self._decode_token_cached`, `result.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 719, "end_line": 758, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L719-L758", "decorators": [], "calls": [ "self._decode_suffix", "text.rfind", "self._decode_token_cached", "result.append", "tok_text.find", "self._is_valid_key_prefix" ], "state_reads": [ "self._decode_suffix", "self._decode_token_cached", "self._valid_key_names", "self._is_valid_key_prefix" ], "state_writes": [], "raises": [], "return_expressions": [ "allowed", "result if result else allowed" ], "awaits": false, "yields": false }, { "name": "_is_valid_key_prefix", "qualname": "JSONSchemaLogitsProcessor._is_valid_key_prefix", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._is_valid_key_prefix", "kind": "method", "signature": "def _is_valid_key_prefix(self, prefix: str) -> bool", "parameters": [ { "name": "prefix", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True if *prefix* is a prefix of at least one valid key name.", "summary": "Return True if *prefix* is a prefix of at least one valid key name.", "implementation": "Method `JSONSchemaLogitsProcessor._is_valid_key_prefix` calls `any`, `name.startswith`; returns `any((name.startswith(prefix) for name in self._valid_key_names))`.", "documented": true, "public": false, "addressable": true, "line": 760, "end_line": 762, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L760-L762", "decorators": [], "calls": [ "any", "name.startswith" ], "state_reads": [ "self._valid_key_names" ], "state_writes": [], "raises": [], "return_expressions": [ "any((name.startswith(prefix) for name in self._valid_key_names))" ], "awaits": false, "yields": false }, { "name": "_filter_nonprogress_whitespace_tokens", "qualname": "JSONSchemaLogitsProcessor._filter_nonprogress_whitespace_tokens", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._filter_nonprogress_whitespace_tokens", "kind": "method", "signature": "def _filter_nonprogress_whitespace_tokens(self, suffix: list[int], allowed: list[int]) -> list[int]", "parameters": [ { "name": "suffix", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "allowed", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[int]", "docstring": "Stop constrained JSON from spending a long run on pure whitespace.\n\nJSON permits arbitrary whitespace around structural tokens. That is\nvalid, but with non-streaming requests a model can keep selecting\nwhitespace-only tokens for minutes without producing useful JSON\ncontent. Once the decoded suffix has a long trailing whitespace run\noutside a string, remove pure-whitespace tokens from the next-step\nallowed set so generation must make structural/content progress.", "summary": "Stop constrained JSON from spending a long run on pure whitespace.", "implementation": "Method `JSONSchemaLogitsProcessor._filter_nonprogress_whitespace_tokens` calls `self._decode_suffix`, `len`, `text.rstrip`, `self._decode_token_cached`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 764, "end_line": 793, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L764-L793", "decorators": [], "calls": [ "self._decode_suffix", "len", "text.rstrip", "self._decode_token_cached", "filtered.append", "all" ], "state_reads": [ "self._decode_suffix", "self._decode_token_cached" ], "state_writes": [], "raises": [], "return_expressions": [ "allowed", "filtered if filtered else allowed" ], "awaits": false, "yields": false }, { "name": "_build_allow_mask", "qualname": "JSONSchemaLogitsProcessor._build_allow_mask", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor._build_allow_mask", "kind": "method", "signature": "def _build_allow_mask(self, allowed: list[int], vocab_size: int) -> mx.array", "parameters": [ { "name": "allowed", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "vocab_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Build a 1-D mask of length ``vocab_size`` where allowed positions are\n``0`` and disallowed positions are ``-inf``.\n\nUses numpy for mask construction (C-level speed) instead of a\nPython loop over ``vocab_size`` elements.", "summary": "Build a 1-D mask of length ``vocab_size`` where allowed positions are ``0`` and disallowed positions are ``-inf``.", "implementation": "Method `JSONSchemaLogitsProcessor._build_allow_mask` calls `mx.full`, `float`, `np.full`, `mx.array`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 795, "end_line": 810, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L795-L810", "decorators": [], "calls": [ "mx.full", "float", "np.full", "mx.array" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.full((vocab_size,), -float('inf'))", "mx.array(buf)" ], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "JSONSchemaLogitsProcessor.__call__", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor.__call__", "kind": "method", "signature": "def __call__(self, tokens: mx.array, logits: mx.array) -> mx.array", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Apply the allowed-tokens mask to ``logits``.", "summary": "Apply the allowed-tokens mask to ``logits``.", "implementation": "Method `JSONSchemaLogitsProcessor.__call__` updates `self._disabled`; calls `_eos_logits_or_original`, `hasattr`, `tokens.tolist`, `list`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 814, "end_line": 910, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L814-L910", "decorators": [], "calls": [ "_eos_logits_or_original", "hasattr", "tokens.tolist", "list", "isinstance", "self._suffix", "_complete_json_eos_logits", "self._enforcer.get_allowed_tokens", "getattr", "self._get_json_context", "self._filter_nonprogress_whitespace_tokens", "self._filter_at_key_context", "any", "self._suffix_is_complete_json", "sorted", "self._tokenizer.decode", "logger.warning", "len", "self._build_allow_mask", "logger.error" ], "state_reads": [ "self._disabled", "self._eos_set", "self._build_allow_mask", "self._suffix", "self._suffix_is_complete_json", "self._prompt_len", "self._enforcer.get_allowed_tokens", "self._enforcer", "self._get_json_context", "self._json_ctx_in_string", "self._filter_nonprogress_whitespace_tokens", "self._filter_at_key_context", "self._tokenizer.decode", "self._tokenizer" ], "state_writes": [ "self._disabled" ], "raises": [], "return_expressions": [ "_eos_logits_or_original(self._eos_set, logits, self._build_allow_mask)", "eos_logits", "logits", "logits + mask" ], "awaits": false, "yields": false }, { "name": "schema", "qualname": "JSONSchemaLogitsProcessor.schema", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor.schema", "kind": "method", "signature": "def schema(self) -> dict | None", "parameters": [], "return_annotation": "dict | None", "docstring": "Return the normalized JSON Schema enforced for this request.", "summary": "Return the normalized JSON Schema enforced for this request.", "implementation": "Method `JSONSchemaLogitsProcessor.schema` returns `self._schema`.", "documented": true, "public": true, "addressable": true, "line": 915, "end_line": 918, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L915-L918", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._schema" ], "state_writes": [], "raises": [], "return_expressions": [ "self._schema" ], "awaits": false, "yields": false }, { "name": "vocab_size", "qualname": "JSONSchemaLogitsProcessor.vocab_size", "full_name": "vllm_mlx.constrained.json_schema_processor.JSONSchemaLogitsProcessor.vocab_size", "kind": "method", "signature": "def vocab_size(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return the tokenizer vocabulary size used to construct masks.", "summary": "Return the tokenizer vocabulary size used to construct masks.", "implementation": "Method `JSONSchemaLogitsProcessor.vocab_size` returns `self._vocab_size`.", "documented": true, "public": true, "addressable": true, "line": 921, "end_line": 924, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/json_schema_processor.py#L921-L924", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._vocab_size" ], "state_writes": [], "raises": [], "return_expressions": [ "self._vocab_size" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.constrained.thinking_processor", "path": "vllm_mlx/constrained/thinking_processor.py", "page_path": "reference/api/vllm_mlx/constrained/thinking_processor.md", "docstring": "Thinking-aware logits processor for reasoning models.\n\nManages the full thinking lifecycle: budget enforcement, phase transitions,\nand content-phase constrained decoding delegation.", "summary": "Thinking-aware logits processor for reasoning models.", "line_count": 287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L1-L287", "members": [ "BoundedSuffixMatcher", "Phase", "ThinkingAwareLogitsProcessor" ], "symbols": [ { "name": "BoundedSuffixMatcher", "qualname": "BoundedSuffixMatcher", "full_name": "vllm_mlx.constrained.thinking_processor.BoundedSuffixMatcher", "kind": "class", "signature": "class BoundedSuffixMatcher", "parameters": [ { "name": "target_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BoundedSuffixMatcher", "docstring": "Detect a target token sequence in a stream using a rolling suffix buffer.\n\nUnlike a naive sequential matcher that resets to position 0 on mismatch,\nthis uses a bounded buffer that catches overlapping prefixes.", "summary": "Detect a target token sequence in a stream using a rolling suffix buffer.", "implementation": "Class `BoundedSuffixMatcher` declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 16, "end_line": 48, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L16-L48", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "BoundedSuffixMatcher.__init__", "full_name": "vllm_mlx.constrained.thinking_processor.BoundedSuffixMatcher.__init__", "kind": "method", "signature": "def __init__(self, target_ids: list[int]) -> None", "parameters": [ { "name": "target_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `BoundedSuffixMatcher.__init__` updates `self.target`, `self._max_len`, `self._buf`; calls `ValueError`, `tuple`, `len`, `deque`; can raise `ValueError`.", "implementation": "Method `BoundedSuffixMatcher.__init__` updates `self.target`, `self._max_len`, `self._buf`; calls `ValueError`, `tuple`, `len`, `deque`; can raise `ValueError`.", "documented": false, "public": false, "addressable": true, "line": 25, "end_line": 30, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L25-L30", "decorators": [], "calls": [ "ValueError", "tuple", "len", "deque" ], "state_reads": [ "self._max_len" ], "state_writes": [ "self.target", "self._max_len", "self._buf" ], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "feed", "qualname": "BoundedSuffixMatcher.feed", "full_name": "vllm_mlx.constrained.thinking_processor.BoundedSuffixMatcher.feed", "kind": "method", "signature": "def feed(self, token_id: int) -> bool", "parameters": [ { "name": "token_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Feed one token. Returns True when the buffer suffix equals the target.", "summary": "Feed one token.", "implementation": "Method `BoundedSuffixMatcher.feed` calls `self._buf.append`, `len`, `tuple`; returns `len(self._buf) == self._max_len and tuple(self._buf) == self.target`.", "documented": true, "public": true, "addressable": true, "line": 32, "end_line": 35, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L32-L35", "decorators": [], "calls": [ "self._buf.append", "len", "tuple" ], "state_reads": [ "self._buf.append", "self._buf", "self._max_len", "self.target" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self._buf) == self._max_len and tuple(self._buf) == self.target" ], "awaits": false, "yields": false }, { "name": "reset", "qualname": "BoundedSuffixMatcher.reset", "full_name": "vllm_mlx.constrained.thinking_processor.BoundedSuffixMatcher.reset", "kind": "method", "signature": "def reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear the buffer.", "summary": "Clear the buffer.", "implementation": "Method `BoundedSuffixMatcher.reset` calls `self._buf.clear`.", "documented": true, "public": true, "addressable": true, "line": 37, "end_line": 39, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L37-L39", "decorators": [], "calls": [ "self._buf.clear" ], "state_reads": [ "self._buf.clear", "self._buf" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "snapshot", "qualname": "BoundedSuffixMatcher.snapshot", "full_name": "vllm_mlx.constrained.thinking_processor.BoundedSuffixMatcher.snapshot", "kind": "method", "signature": "def snapshot(self) -> tuple[int, ...]", "parameters": [], "return_annotation": "tuple[int, ...]", "docstring": "Return a serializable copy of the current suffix buffer.", "summary": "Return a serializable copy of the current suffix buffer.", "implementation": "Method `BoundedSuffixMatcher.snapshot` calls `tuple`; returns `tuple(self._buf)`.", "documented": true, "public": true, "addressable": true, "line": 41, "end_line": 43, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L41-L43", "decorators": [], "calls": [ "tuple" ], "state_reads": [ "self._buf" ], "state_writes": [], "raises": [], "return_expressions": [ "tuple(self._buf)" ], "awaits": false, "yields": false }, { "name": "restore", "qualname": "BoundedSuffixMatcher.restore", "full_name": "vllm_mlx.constrained.thinking_processor.BoundedSuffixMatcher.restore", "kind": "method", "signature": "def restore(self, state: tuple[int, ...]) -> None", "parameters": [ { "name": "state", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Restore the suffix buffer from a previous snapshot.", "summary": "Restore the suffix buffer from a previous snapshot.", "implementation": "Method `BoundedSuffixMatcher.restore` calls `self._buf.clear`, `self._buf.extend`.", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 48, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L45-L48", "decorators": [], "calls": [ "self._buf.clear", "self._buf.extend" ], "state_reads": [ "self._buf.clear", "self._buf", "self._buf.extend" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "Phase", "qualname": "Phase", "full_name": "vllm_mlx.constrained.thinking_processor.Phase", "kind": "class", "signature": "class Phase(enum.Enum)", "parameters": [], "return_annotation": "Phase", "docstring": "Thinking lifecycle phases.", "summary": "Thinking lifecycle phases.", "implementation": "Class `Phase` derives from `enum.Enum` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 51, "end_line": 57, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L51-L57", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ThinkingAwareLogitsProcessor", "qualname": "ThinkingAwareLogitsProcessor", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor", "kind": "class", "signature": "class ThinkingAwareLogitsProcessor", "parameters": [ { "name": "start_token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "end_token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "thinking_token_budget", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "inner", "kind": "positional or keyword", "annotation": "Callable[[mx.array, mx.array], mx.array] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "vocab_size", "kind": "positional or keyword", "annotation": "int", "default": "152064", "required": false, "description": "Optional positional or keyword input; defaults to `152064`." }, { "name": "prompt_has_think_tag", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." }, { "name": "no_final_content_token_limit", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ThinkingAwareLogitsProcessor", "docstring": "Unified logits processor for thinking-model lifecycle management.\n\nManages a four-phase state machine:\n IDLE -> THINKING -> TRANSITIONING -> CONTENT\n\n- IDLE: before reasoning start tokens. Pass through.\n- THINKING: inside reasoning span. Count tokens, pass through.\n- TRANSITIONING: forcing reasoning end sequence via logits masking.\n- CONTENT: after reasoning closed. Delegate to inner processor.\n\nNo re-entry into THINKING after CONTENT is reached.", "summary": "Unified logits processor for thinking-model lifecycle management.", "implementation": "Class `ThinkingAwareLogitsProcessor` declares 12 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 60, "end_line": 287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L60-L287", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ThinkingAwareLogitsProcessor.__init__", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor.__init__", "kind": "method", "signature": "def __init__(self, start_token_ids: list[int], end_token_ids: list[int], thinking_token_budget: int, inner: Callable[[mx.array, mx.array], mx.array] | None=None, vocab_size: int=152064, prompt_has_think_tag: bool=False, no_final_content_token_limit: int | None=None) -> None", "parameters": [ { "name": "start_token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "end_token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "thinking_token_budget", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "inner", "kind": "positional or keyword", "annotation": "Callable[[mx.array, mx.array], mx.array] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "vocab_size", "kind": "positional or keyword", "annotation": "int", "default": "152064", "required": false, "description": "Optional positional or keyword input; defaults to `152064`." }, { "name": "prompt_has_think_tag", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." }, { "name": "no_final_content_token_limit", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ThinkingAwareLogitsProcessor.__init__` updates `self._start_matcher`, `self._end_matcher`, `self._end_token_ids`, `self._content_phase_mask_ids`; calls `BoundedSuffixMatcher`, `list`, `tuple`, `dict.fromkeys`.", "implementation": "Method `ThinkingAwareLogitsProcessor.__init__` updates `self._start_matcher`, `self._end_matcher`, `self._end_token_ids`, `self._content_phase_mask_ids`; calls `BoundedSuffixMatcher`, `list`, `tuple`, `dict.fromkeys`.", "documented": false, "public": false, "addressable": true, "line": 92, "end_line": 129, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L92-L129", "decorators": [], "calls": [ "BoundedSuffixMatcher", "list", "tuple", "dict.fromkeys", "self._snapshot_state" ], "state_reads": [ "self._snapshot_state" ], "state_writes": [ "self._start_matcher", "self._end_matcher", "self._end_token_ids", "self._content_phase_mask_ids", "self._thinking_token_budget", "self._inner", "self._vocab_size", "self._thinking_tokens", "self._transition_index", "self.watchdog_was_enforced", "self._no_final_content_token_limit", "self._state", "self._processed_len", "self._processed_token_ids", "self._snapshots" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "state", "qualname": "ThinkingAwareLogitsProcessor.state", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor.state", "kind": "method", "signature": "def state(self) -> Phase", "parameters": [], "return_annotation": "Phase", "docstring": "Return the current reasoning lifecycle phase.", "summary": "Return the current reasoning lifecycle phase.", "implementation": "Method `ThinkingAwareLogitsProcessor.state` returns `self._state`.", "documented": true, "public": true, "addressable": true, "line": 132, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L132-L135", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._state" ], "state_writes": [], "raises": [], "return_expressions": [ "self._state" ], "awaits": false, "yields": false }, { "name": "thinking_tokens", "qualname": "ThinkingAwareLogitsProcessor.thinking_tokens", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor.thinking_tokens", "kind": "method", "signature": "def thinking_tokens(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return the number of generated tokens counted as reasoning.", "summary": "Return the number of generated tokens counted as reasoning.", "implementation": "Method `ThinkingAwareLogitsProcessor.thinking_tokens` returns `self._thinking_tokens`.", "documented": true, "public": true, "addressable": true, "line": 138, "end_line": 141, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L138-L141", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._thinking_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "self._thinking_tokens" ], "awaits": false, "yields": false }, { "name": "is_retired", "qualname": "ThinkingAwareLogitsProcessor.is_retired", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor.is_retired", "kind": "method", "signature": "def is_retired(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "True when the processor is in CONTENT with no inner constraint.\n\nThe engine can use this signal to drop the processor and re-enable\nMTP for the remaining content generation (Phase 2 optimization).", "summary": "True when the processor is in CONTENT with no inner constraint.", "implementation": "Method `ThinkingAwareLogitsProcessor.is_retired` returns `self._state == Phase.CONTENT and self._inner is None`.", "documented": true, "public": true, "addressable": true, "line": 144, "end_line": 150, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L144-L150", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._state", "self._inner" ], "state_writes": [], "raises": [], "return_expressions": [ "self._state == Phase.CONTENT and self._inner is None" ], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "ThinkingAwareLogitsProcessor.__call__", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor.__call__", "kind": "method", "signature": "def __call__(self, tokens: mx.array, logits: mx.array) -> mx.array", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "", "summary": "Method `ThinkingAwareLogitsProcessor.__call__` calls `self._force_transition`, `self._call_inner`, `self._sync_to_tokens`; has 3 explicit return paths.", "implementation": "Method `ThinkingAwareLogitsProcessor.__call__` calls `self._force_transition`, `self._call_inner`, `self._sync_to_tokens`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 152, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L152-L170", "decorators": [], "calls": [ "self._force_transition", "self._call_inner", "self._sync_to_tokens" ], "state_reads": [ "self._state", "self._force_transition", "self._call_inner", "self._sync_to_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "self._force_transition(logits)", "self._call_inner(tokens, logits)", "logits" ], "awaits": false, "yields": false }, { "name": "_force_transition", "qualname": "ThinkingAwareLogitsProcessor._force_transition", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor._force_transition", "kind": "method", "signature": "def _force_transition(self, logits: mx.array) -> mx.array", "parameters": [ { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Force the next token in the reasoning end sequence.", "summary": "Force the next token in the reasoning end sequence.", "implementation": "Method `ThinkingAwareLogitsProcessor._force_transition` calls `mx.full`, `float`; returns `masked`.", "documented": true, "public": false, "addressable": true, "line": 172, "end_line": 182, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L172-L182", "decorators": [], "calls": [ "mx.full", "float" ], "state_reads": [ "self._end_token_ids", "self._transition_index" ], "state_writes": [], "raises": [], "return_expressions": [ "masked" ], "awaits": false, "yields": false }, { "name": "_call_inner", "qualname": "ThinkingAwareLogitsProcessor._call_inner", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor._call_inner", "kind": "method", "signature": "def _call_inner(self, tokens: mx.array, logits: mx.array) -> mx.array", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Delegate to inner processor if present.", "summary": "Delegate to inner processor if present.", "implementation": "Method `ThinkingAwareLogitsProcessor._call_inner` calls `self._inner`, `self._mask_content_phase_control_tokens`; returns `self._mask_content_phase_control_tokens(logits)`.", "documented": true, "public": false, "addressable": true, "line": 184, "end_line": 188, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L184-L188", "decorators": [], "calls": [ "self._inner", "self._mask_content_phase_control_tokens" ], "state_reads": [ "self._inner", "self._mask_content_phase_control_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "self._mask_content_phase_control_tokens(logits)" ], "awaits": false, "yields": false }, { "name": "_mask_content_phase_control_tokens", "qualname": "ThinkingAwareLogitsProcessor._mask_content_phase_control_tokens", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor._mask_content_phase_control_tokens", "kind": "method", "signature": "def _mask_content_phase_control_tokens(self, logits: mx.array) -> mx.array", "parameters": [ { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Prevent reserved think-tag starts from leaking into final content.", "summary": "Prevent reserved think-tag starts from leaking into final content.", "implementation": "Method `ThinkingAwareLogitsProcessor._mask_content_phase_control_tokens` calls `float`; returns `logits`.", "documented": true, "public": false, "addressable": true, "line": 190, "end_line": 197, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L190-L197", "decorators": [], "calls": [ "float" ], "state_reads": [ "self._content_phase_mask_ids" ], "state_writes": [], "raises": [], "return_expressions": [ "logits" ], "awaits": false, "yields": false }, { "name": "_snapshot_state", "qualname": "ThinkingAwareLogitsProcessor._snapshot_state", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor._snapshot_state", "kind": "method", "signature": "def _snapshot_state(self) -> tuple[Phase, int, int, tuple[int, ...], tuple[int, ...], bool]", "parameters": [], "return_annotation": "tuple[Phase, int, int, tuple[int, ...], tuple[int, ...], bool]", "docstring": "", "summary": "Method `ThinkingAwareLogitsProcessor._snapshot_state` calls `self._start_matcher.snapshot`, `self._end_matcher.snapshot`; returns `(self._state, self._thinking_tokens, self._transition_index, self._start_matcher.snapshot(), self._end_matcher.snapshot…`.", "implementation": "Method `ThinkingAwareLogitsProcessor._snapshot_state` calls `self._start_matcher.snapshot`, `self._end_matcher.snapshot`; returns `(self._state, self._thinking_tokens, self._transition_index, self._start_matcher.snapshot(), self._end_matcher.snapshot…`.", "documented": false, "public": false, "addressable": true, "line": 199, "end_line": 209, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L199-L209", "decorators": [], "calls": [ "self._start_matcher.snapshot", "self._end_matcher.snapshot" ], "state_reads": [ "self._state", "self._thinking_tokens", "self._transition_index", "self._start_matcher.snapshot", "self._start_matcher", "self._end_matcher.snapshot", "self._end_matcher", "self.watchdog_was_enforced" ], "state_writes": [], "raises": [], "return_expressions": [ "(self._state, self._thinking_tokens, self._transition_index, self._start_matcher.snapshot(), self._end_matcher.snapshot…" ], "awaits": false, "yields": false }, { "name": "_restore_snapshot", "qualname": "ThinkingAwareLogitsProcessor._restore_snapshot", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor._restore_snapshot", "kind": "method", "signature": "def _restore_snapshot(self, processed_len: int) -> None", "parameters": [ { "name": "processed_len", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ThinkingAwareLogitsProcessor._restore_snapshot` updates `self._state`, `self._thinking_tokens`, `self._transition_index`, `self.watchdog_was_enforced`; calls `min`, `len`, `self._start_matcher.restore`, `self._end_matcher.restore`.", "implementation": "Method `ThinkingAwareLogitsProcessor._restore_snapshot` updates `self._state`, `self._thinking_tokens`, `self._transition_index`, `self.watchdog_was_enforced`; calls `min`, `len`, `self._start_matcher.restore`, `self._end_matcher.restore`.", "documented": false, "public": false, "addressable": true, "line": 211, "end_line": 229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L211-L229", "decorators": [], "calls": [ "min", "len", "self._start_matcher.restore", "self._end_matcher.restore" ], "state_reads": [ "self._snapshots", "self._start_matcher.restore", "self._start_matcher", "self._end_matcher.restore", "self._end_matcher", "self._processed_token_ids" ], "state_writes": [ "self._state", "self._thinking_tokens", "self._transition_index", "self.watchdog_was_enforced", "self._processed_len", "self._processed_token_ids", "self._snapshots" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_sync_to_tokens", "qualname": "ThinkingAwareLogitsProcessor._sync_to_tokens", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor._sync_to_tokens", "kind": "method", "signature": "def _sync_to_tokens(self, tokens: mx.array) -> None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ThinkingAwareLogitsProcessor._sync_to_tokens` updates `self._processed_len`; calls `int`, `tokens.tolist`, `min`, `self._restore_snapshot`; returns `None`.", "implementation": "Method `ThinkingAwareLogitsProcessor._sync_to_tokens` updates `self._processed_len`; calls `int`, `tokens.tolist`, `min`, `self._restore_snapshot`; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 231, "end_line": 252, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L231-L252", "decorators": [], "calls": [ "int", "tokens.tolist", "min", "self._restore_snapshot", "self._advance_with_token", "self._processed_token_ids.append", "self._snapshots.append", "self._snapshot_state" ], "state_reads": [ "self._processed_len", "self._processed_token_ids", "self._restore_snapshot", "self._advance_with_token", "self._processed_token_ids.append", "self._state", "self._snapshots.append", "self._snapshots", "self._snapshot_state" ], "state_writes": [ "self._processed_len" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_advance_with_token", "qualname": "ThinkingAwareLogitsProcessor._advance_with_token", "full_name": "vllm_mlx.constrained.thinking_processor.ThinkingAwareLogitsProcessor._advance_with_token", "kind": "method", "signature": "def _advance_with_token(self, token_id: int) -> None", "parameters": [ { "name": "token_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ThinkingAwareLogitsProcessor._advance_with_token` updates `self._state`, `self._transition_index`, `self._thinking_tokens`, `self.watchdog_was_enforced`; calls `self._start_matcher.feed`, `self._end_matcher.feed`, `len`, `self._end_matcher.reset`; returns `None`.", "implementation": "Method `ThinkingAwareLogitsProcessor._advance_with_token` updates `self._state`, `self._transition_index`, `self._thinking_tokens`, `self.watchdog_was_enforced`; calls `self._start_matcher.feed`, `self._end_matcher.feed`, `len`, `self._end_matcher.reset`; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 254, "end_line": 287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/constrained/thinking_processor.py#L254-L287", "decorators": [], "calls": [ "self._start_matcher.feed", "self._end_matcher.feed", "len", "self._end_matcher.reset" ], "state_reads": [ "self._state", "self._start_matcher.feed", "self._start_matcher", "self._thinking_token_budget", "self._end_matcher.feed", "self._end_matcher", "self._thinking_tokens", "self._no_final_content_token_limit", "self._end_token_ids", "self._transition_index", "self._end_matcher.reset" ], "state_writes": [ "self._state", "self._transition_index", "self._thinking_tokens", "self.watchdog_was_enforced" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.embedding", "path": "vllm_mlx/embedding.py", "page_path": "reference/api/vllm_mlx/embedding.md", "docstring": "Embedding engine using mlx-embeddings.\n\nProvides lazy-loaded model management and batch embedding generation\nfor the OpenAI-compatible /v1/embeddings endpoint.", "summary": "Embedding engine using mlx-embeddings.", "line_count": 131, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L1-L131", "members": [ "logger", "EmbeddingEngine" ], "symbols": [ { "name": "EmbeddingEngine", "qualname": "EmbeddingEngine", "full_name": "vllm_mlx.embedding.EmbeddingEngine", "kind": "class", "signature": "class EmbeddingEngine", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "EmbeddingEngine", "docstring": "Wrapper around mlx-embeddings for text embedding generation.\n\nSupports lazy model loading and batch embedding with proper\ntokenization and pooling.", "summary": "Wrapper around mlx-embeddings for text embedding generation.", "implementation": "Class `EmbeddingEngine` declares 7 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 19, "end_line": 131, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L19-L131", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "EmbeddingEngine.__init__", "full_name": "vllm_mlx.embedding.EmbeddingEngine.__init__", "kind": "method", "signature": "def __init__(self, model_name: str)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `EmbeddingEngine.__init__` updates `self.model_name`, `self._model`, `self._tokenizer`, `self._max_length`.", "implementation": "Method `EmbeddingEngine.__init__` updates `self.model_name`, `self._model`, `self._tokenizer`, `self._max_length`.", "documented": false, "public": false, "addressable": true, "line": 27, "end_line": 31, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L27-L31", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.model_name", "self._model", "self._tokenizer", "self._max_length" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_loaded", "qualname": "EmbeddingEngine.is_loaded", "full_name": "vllm_mlx.embedding.EmbeddingEngine.is_loaded", "kind": "method", "signature": "def is_loaded(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return whether the embedding model has been loaded.", "summary": "Return whether the embedding model has been loaded.", "implementation": "Method `EmbeddingEngine.is_loaded` returns `self._model is not None`.", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L34-L37", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "self._model is not None" ], "awaits": false, "yields": false }, { "name": "load", "qualname": "EmbeddingEngine.load", "full_name": "vllm_mlx.embedding.EmbeddingEngine.load", "kind": "method", "signature": "def load(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the embedding model and tokenizer.", "summary": "Load the embedding model and tokenizer.", "implementation": "Method `EmbeddingEngine.load` updates `self._model`, `self._tokenizer`; calls `logger.info`, `time.perf_counter`, `load`.", "documented": true, "public": true, "addressable": true, "line": 39, "end_line": 47, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L39-L47", "decorators": [], "calls": [ "logger.info", "time.perf_counter", "load" ], "state_reads": [ "self.model_name" ], "state_writes": [ "self._model", "self._tokenizer" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_ensure_loaded", "qualname": "EmbeddingEngine._ensure_loaded", "full_name": "vllm_mlx.embedding.EmbeddingEngine._ensure_loaded", "kind": "method", "signature": "def _ensure_loaded(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `EmbeddingEngine._ensure_loaded` calls `self.load`.", "implementation": "Method `EmbeddingEngine._ensure_loaded` calls `self.load`.", "documented": false, "public": false, "addressable": true, "line": 49, "end_line": 51, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L49-L51", "decorators": [], "calls": [ "self.load" ], "state_reads": [ "self.is_loaded", "self.load" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_resolve_max_length", "qualname": "EmbeddingEngine._resolve_max_length", "full_name": "vllm_mlx.embedding.EmbeddingEngine._resolve_max_length", "kind": "method", "signature": "def _resolve_max_length(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Tokenizer truncation length from the model config (cached).", "summary": "Tokenizer truncation length from the model config (cached).", "implementation": "Method `EmbeddingEngine._resolve_max_length` updates `self._max_length`; calls `resolve_max_length`, `getattr`; returns `self._max_length`.", "documented": true, "public": false, "addressable": true, "line": 53, "end_line": 60, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L53-L60", "decorators": [], "calls": [ "resolve_max_length", "getattr" ], "state_reads": [ "self._max_length", "self._model", "self._tokenizer" ], "state_writes": [ "self._max_length" ], "raises": [], "return_expressions": [ "self._max_length" ], "awaits": false, "yields": false }, { "name": "embed", "qualname": "EmbeddingEngine.embed", "full_name": "vllm_mlx.embedding.EmbeddingEngine.embed", "kind": "method", "signature": "def embed(self, texts: str | list[str]) -> list[list[float]]", "parameters": [ { "name": "texts", "kind": "positional or keyword", "annotation": "str | list[str]", "default": "", "required": true, "description": "A single string or list of strings." } ], "return_annotation": "list[list[float]]", "docstring": "Generate embeddings for one or more texts.\n\nArgs:\n texts: A single string or list of strings.\n\nReturns:\n List of embedding vectors (one per input text).", "summary": "Generate embeddings for one or more texts.", "implementation": "Method `EmbeddingEngine.embed` calls `self._ensure_loaded`, `isinstance`, `inner_tokenizer`, `inner_tok`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 62, "end_line": 109, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L62-L109", "decorators": [], "calls": [ "self._ensure_loaded", "isinstance", "inner_tokenizer", "inner_tok", "self._resolve_max_length", "mx.array", "self._model", "embeds.tolist", "mx.clear_cache" ], "state_reads": [ "self._ensure_loaded", "self._tokenizer", "self._resolve_max_length", "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "count_tokens", "qualname": "EmbeddingEngine.count_tokens", "full_name": "vllm_mlx.embedding.EmbeddingEngine.count_tokens", "kind": "method", "signature": "def count_tokens(self, texts: str | list[str]) -> int", "parameters": [ { "name": "texts", "kind": "positional or keyword", "annotation": "str | list[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Approximate token count for usage reporting.", "summary": "Approximate token count for usage reporting.", "implementation": "Method `EmbeddingEngine.count_tokens` calls `self._ensure_loaded`, `isinstance`, `self._tokenizer.encode`, `len`; returns `total`.", "documented": true, "public": true, "addressable": true, "line": 111, "end_line": 131, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/embedding.py#L111-L131", "decorators": [], "calls": [ "self._ensure_loaded", "isinstance", "self._tokenizer.encode", "len", "hasattr", "max" ], "state_reads": [ "self._ensure_loaded", "self._tokenizer.encode", "self._tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "total" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.endpoint_model_policies", "path": "vllm_mlx/endpoint_model_policies.py", "page_path": "reference/api/vllm_mlx/endpoint_model_policies.md", "docstring": "Request-time model resolution policies for optional endpoints.\n\nThese endpoints intentionally do not expose arbitrary Hugging Face loading from\nuser-controlled request bodies. Unknown model names must be rejected before any\nengine instantiation or download path is reached.", "summary": "Request-time model resolution policies for optional endpoints.", "line_count": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/endpoint_model_policies.py#L1-L118", "members": [ "_EMBEDDING_MODELS", "_STT_MODEL_ALIASES", "_TTS_MODEL_ALIASES", "_with_identity_aliases", "_STT_MODEL_MAP", "_TTS_MODEL_MAP", "_reject_unknown_embedding_model", "_reject_unknown_audio_model", "resolve_embedding_model_name", "resolve_stt_model_name", "resolve_tts_model_name" ], "symbols": [ { "name": "_with_identity_aliases", "qualname": "_with_identity_aliases", "full_name": "vllm_mlx.endpoint_model_policies._with_identity_aliases", "kind": "function", "signature": "def _with_identity_aliases(model_map: dict[str, str]) -> dict[str, str]", "parameters": [ { "name": "model_map", "kind": "positional or keyword", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, str]", "docstring": "", "summary": "Function `_with_identity_aliases` calls `dict`, `model_map.values`; returns `expanded`.", "implementation": "Function `_with_identity_aliases` calls `dict`, `model_map.values`; returns `expanded`.", "documented": false, "public": false, "addressable": true, "line": 42, "end_line": 46, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/endpoint_model_policies.py#L42-L46", "decorators": [], "calls": [ "dict", "model_map.values" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "expanded" ], "awaits": false, "yields": false }, { "name": "_reject_unknown_embedding_model", "qualname": "_reject_unknown_embedding_model", "full_name": "vllm_mlx.endpoint_model_policies._reject_unknown_embedding_model", "kind": "function", "signature": "def _reject_unknown_embedding_model(requested_model: str) -> None", "parameters": [ { "name": "requested_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Function `_reject_unknown_embedding_model` calls `', '.join`, `sorted`, `HTTPException`; can raise `HTTPException`.", "implementation": "Function `_reject_unknown_embedding_model` calls `', '.join`, `sorted`, `HTTPException`; can raise `HTTPException`.", "documented": false, "public": false, "addressable": true, "line": 53, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/endpoint_model_policies.py#L53-L63", "decorators": [], "calls": [ "', '.join", "sorted", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_reject_unknown_audio_model", "qualname": "_reject_unknown_audio_model", "full_name": "vllm_mlx.endpoint_model_policies._reject_unknown_audio_model", "kind": "function", "signature": "def _reject_unknown_audio_model(endpoint: str, requested_model: str, supported_aliases: dict[str, str]) -> None", "parameters": [ { "name": "endpoint", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "requested_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "supported_aliases", "kind": "positional or keyword", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Function `_reject_unknown_audio_model` calls `', '.join`, `sorted`, `HTTPException`; can raise `HTTPException`.", "implementation": "Function `_reject_unknown_audio_model` calls `', '.join`, `sorted`, `HTTPException`; can raise `HTTPException`.", "documented": false, "public": false, "addressable": true, "line": 66, "end_line": 79, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/endpoint_model_policies.py#L66-L79", "decorators": [], "calls": [ "', '.join", "sorted", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "resolve_embedding_model_name", "qualname": "resolve_embedding_model_name", "full_name": "vllm_mlx.endpoint_model_policies.resolve_embedding_model_name", "kind": "function", "signature": "def resolve_embedding_model_name(requested_model: str, *, locked_model: str | None=None) -> str", "parameters": [ { "name": "requested_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "locked_model", "kind": "keyword-only", "annotation": "str | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "str", "docstring": "Resolve the embedding model for a request or raise HTTP 400.", "summary": "Resolve the embedding model for a request or raise HTTP 400.", "implementation": "Function `resolve_embedding_model_name` calls `HTTPException`, `_reject_unknown_embedding_model`; can raise `HTTPException`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 82, "end_line": 104, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/endpoint_model_policies.py#L82-L104", "decorators": [], "calls": [ "HTTPException", "_reject_unknown_embedding_model" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "locked_model", "requested_model" ], "awaits": false, "yields": false }, { "name": "resolve_stt_model_name", "qualname": "resolve_stt_model_name", "full_name": "vllm_mlx.endpoint_model_policies.resolve_stt_model_name", "kind": "function", "signature": "def resolve_stt_model_name(requested_model: str) -> str", "parameters": [ { "name": "requested_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Resolve an STT request model alias or configured model ID.", "summary": "Resolve an STT request model alias or configured model ID.", "implementation": "Function `resolve_stt_model_name` calls `_reject_unknown_audio_model`; returns `_STT_MODEL_MAP[requested_model]`.", "documented": true, "public": true, "addressable": true, "line": 107, "end_line": 111, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/endpoint_model_policies.py#L107-L111", "decorators": [], "calls": [ "_reject_unknown_audio_model" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_STT_MODEL_MAP[requested_model]" ], "awaits": false, "yields": false }, { "name": "resolve_tts_model_name", "qualname": "resolve_tts_model_name", "full_name": "vllm_mlx.endpoint_model_policies.resolve_tts_model_name", "kind": "function", "signature": "def resolve_tts_model_name(requested_model: str) -> str", "parameters": [ { "name": "requested_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Resolve a TTS request model alias or configured model ID.", "summary": "Resolve a TTS request model alias or configured model ID.", "implementation": "Function `resolve_tts_model_name` calls `_reject_unknown_audio_model`; returns `_TTS_MODEL_MAP[requested_model]`.", "documented": true, "public": true, "addressable": true, "line": 114, "end_line": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/endpoint_model_policies.py#L114-L118", "decorators": [], "calls": [ "_reject_unknown_audio_model" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_TTS_MODEL_MAP[requested_model]" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.engine", "path": "vllm_mlx/engine/__init__.py", "page_path": "reference/api/vllm_mlx/engine/index.md", "docstring": "Engine abstraction for vllm-mlx inference.\n\nThe package stays intentionally light at import time so server- and\ncontract-level tests can import API modules without eagerly importing MLX,\nengine_core, or the batched engine stack.", "summary": "Engine abstraction for vllm-mlx inference.", "line_count": 54, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/__init__.py#L1-L54", "members": [ "_ENGINE_CORE_NAMES", "__all__", "__getattr__" ], "symbols": [ { "name": "__getattr__", "qualname": "__getattr__", "full_name": "vllm_mlx.engine.__getattr__", "kind": "function", "signature": "def __getattr__(name: str)", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Function `__getattr__` calls `AttributeError`; can raise `AttributeError`; has 3 explicit return paths.", "implementation": "Function `__getattr__` calls `AttributeError`; can raise `AttributeError`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 34, "end_line": 54, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/__init__.py#L34-L54", "decorators": [], "calls": [ "AttributeError" ], "state_reads": [], "state_writes": [], "raises": [ "AttributeError" ], "return_expressions": [ "SimpleEngine", "BatchedEngine", "{'EngineCore': EngineCore, 'AsyncEngineCore': AsyncEngineCore, 'EngineConfig': EngineConfig}[name]" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.engine.base", "path": "vllm_mlx/engine/base.py", "page_path": "reference/api/vllm_mlx/engine/base.md", "docstring": "Base engine interface for vllm-mlx inference.", "summary": "Base engine interface for vllm-mlx inference.", "line_count": 288, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L1-L288", "members": [ "logger", "GenerationOutput", "EngineBusy", "suspend_cancellation", "run_blocking_startup_work", "cleanup_startup_cancellation", "BaseEngine" ], "symbols": [ { "name": "GenerationOutput", "qualname": "GenerationOutput", "full_name": "vllm_mlx.engine.base.GenerationOutput", "kind": "class", "signature": "class GenerationOutput", "parameters": [ { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens", "kind": "field", "annotation": "list[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "completion_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "finish_reason", "kind": "field", "annotation": "str | None", "default": "'stop'", "required": false, "description": "Optional constructor field; defaults to `'stop'`." }, { "name": "mtp_drafts", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_accepted", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "new_text", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "finished", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "mtp_drafts", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_accepted", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "GenerationOutput", "docstring": "Output from generation.\n\nCompatible with both simple and batched engines.", "summary": "Output from generation.", "implementation": "Class `GenerationOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 18, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L18-L37", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "EngineBusy", "qualname": "EngineBusy", "full_name": "vllm_mlx.engine.base.EngineBusy", "kind": "class", "signature": "class EngineBusy(RuntimeError)", "parameters": [], "return_annotation": "EngineBusy", "docstring": "Raised when a serialized engine route is already serving a request.", "summary": "Raised when a serialized engine route is already serving a request.", "implementation": "Class `EngineBusy` derives from `RuntimeError` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 43, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L40-L43", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "suspend_cancellation", "qualname": "suspend_cancellation", "full_name": "vllm_mlx.engine.base.suspend_cancellation", "kind": "function", "signature": "def suspend_cancellation()", "parameters": [], "return_annotation": "not annotated", "docstring": "Temporarily clear task cancellation so cleanup can finish deterministically.", "summary": "Temporarily clear task cancellation so cleanup can finish deterministically.", "implementation": "Function `suspend_cancellation` calls `asyncio.current_task`, `getattr`, `cancelling`, `range`; yields values incrementally; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 47, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L47-L67", "decorators": [ "contextmanager" ], "calls": [ "asyncio.current_task", "getattr", "cancelling", "range", "uncancel", "task.cancel" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": true }, { "name": "run_blocking_startup_work", "qualname": "run_blocking_startup_work", "full_name": "vllm_mlx.engine.base.run_blocking_startup_work", "kind": "function", "signature": "async def run_blocking_startup_work(work: Callable[[], Any]) -> None", "parameters": [ { "name": "work", "kind": "positional or keyword", "annotation": "Callable[[], Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Run blocking startup work off-loop without leaking cancellation races.", "summary": "Run blocking startup work off-loop without leaking cancellation races.", "implementation": "Function `run_blocking_startup_work` calls `asyncio.create_task`, `asyncio.to_thread`, `asyncio.shield`, `suspend_cancellation`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 84, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L70-L84", "decorators": [], "calls": [ "asyncio.create_task", "asyncio.to_thread", "asyncio.shield", "suspend_cancellation", "task.done" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "cleanup_startup_cancellation", "qualname": "cleanup_startup_cancellation", "full_name": "vllm_mlx.engine.base.cleanup_startup_cancellation", "kind": "function", "signature": "async def cleanup_startup_cancellation(cleanup: Callable[[], Awaitable[None]]) -> None", "parameters": [ { "name": "cleanup", "kind": "positional or keyword", "annotation": "Callable[[], Awaitable[None]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Run startup cleanup without letting cleanup failures replace cancellation.", "summary": "Run startup cleanup without letting cleanup failures replace cancellation.", "implementation": "Function `cleanup_startup_cancellation` calls `suspend_cancellation`, `cleanup`, `isinstance`, `logger.error`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 87, "end_line": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L87-L98", "decorators": [], "calls": [ "suspend_cancellation", "cleanup", "isinstance", "logger.error", "type" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "BaseEngine", "qualname": "BaseEngine", "full_name": "vllm_mlx.engine.base.BaseEngine", "kind": "class", "signature": "class BaseEngine(ABC)", "parameters": [], "return_annotation": "BaseEngine", "docstring": "Abstract base class for inference engines.\n\nBoth SimpleEngine and BatchedEngine implement this interface,\nallowing the server to use either without code changes.", "summary": "Abstract base class for inference engines.", "implementation": "Class `BaseEngine` derives from `ABC` and declares 16 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 101, "end_line": 288, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L101-L288", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "model_name", "qualname": "BaseEngine.model_name", "full_name": "vllm_mlx.engine.base.BaseEngine.model_name", "kind": "method", "signature": "def model_name(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Get the model name.", "summary": "Get the model name.", "implementation": "Method `BaseEngine.model_name` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 111, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L111-L113", "decorators": [ "property", "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_mllm", "qualname": "BaseEngine.is_mllm", "full_name": "vllm_mlx.engine.base.BaseEngine.is_mllm", "kind": "method", "signature": "def is_mllm(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if this is a multimodal model.", "summary": "Check if this is a multimodal model.", "implementation": "Method `BaseEngine.is_mllm` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 117, "end_line": 119, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L117-L119", "decorators": [ "property", "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "tokenizer", "qualname": "BaseEngine.tokenizer", "full_name": "vllm_mlx.engine.base.BaseEngine.tokenizer", "kind": "method", "signature": "def tokenizer(self) -> Any", "parameters": [], "return_annotation": "Any", "docstring": "Get the tokenizer.", "summary": "Get the tokenizer.", "implementation": "Method `BaseEngine.tokenizer` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 123, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L123-L125", "decorators": [ "property", "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "preserve_native_tool_format", "qualname": "BaseEngine.preserve_native_tool_format", "full_name": "vllm_mlx.engine.base.BaseEngine.preserve_native_tool_format", "kind": "method", "signature": "def preserve_native_tool_format(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Whether to preserve native tool message format.\n\nWhen True, role=\"tool\" messages and tool_calls fields are preserved\ninstead of being converted to text. Set by server based on tool parser.", "summary": "Whether to preserve native tool message format.", "implementation": "Method `BaseEngine.preserve_native_tool_format` calls `getattr`; returns `getattr(self, '_preserve_native_tool_format', False)`.", "documented": true, "public": true, "addressable": true, "line": 128, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L128-L135", "decorators": [ "property" ], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "getattr(self, '_preserve_native_tool_format', False)" ], "awaits": false, "yields": false }, { "name": "preserve_native_tool_format", "qualname": "BaseEngine.preserve_native_tool_format", "full_name": "vllm_mlx.engine.base.BaseEngine.preserve_native_tool_format", "kind": "method", "signature": "def preserve_native_tool_format(self, value: bool) -> None", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Enable or disable preservation of model-native tool messages.", "summary": "Enable or disable preservation of model-native tool messages.", "implementation": "Method `BaseEngine.preserve_native_tool_format` updates `self._preserve_native_tool_format`.", "documented": true, "public": true, "addressable": true, "line": 138, "end_line": 141, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L138-L141", "decorators": [ "preserve_native_tool_format.setter" ], "calls": [], "state_reads": [], "state_writes": [ "self._preserve_native_tool_format" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "prepare_for_start", "qualname": "BaseEngine.prepare_for_start", "full_name": "vllm_mlx.engine.base.BaseEngine.prepare_for_start", "kind": "method", "signature": "def prepare_for_start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Run blocking startup work before async engine start.\n\nEngines can override this to perform heavyweight synchronous model\nloads off the serving event loop. The default implementation is a\nno-op so lightweight engines do not need extra plumbing.", "summary": "Run blocking startup work before async engine start.", "implementation": "Method `BaseEngine.prepare_for_start` returns `None`.", "documented": true, "public": true, "addressable": true, "line": 143, "end_line": 150, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L143-L150", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "start", "qualname": "BaseEngine.start", "full_name": "vllm_mlx.engine.base.BaseEngine.start", "kind": "method", "signature": "async def start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the engine (load model if not loaded).", "summary": "Start the engine (load model if not loaded).", "implementation": "Method `BaseEngine.start` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 153, "end_line": 155, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L153-L155", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "stop", "qualname": "BaseEngine.stop", "full_name": "vllm_mlx.engine.base.BaseEngine.stop", "kind": "method", "signature": "async def stop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop the engine and cleanup resources.", "summary": "Stop the engine and cleanup resources.", "implementation": "Method `BaseEngine.stop` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 158, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L158-L160", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "generate", "qualname": "BaseEngine.generate", "full_name": "vllm_mlx.engine.base.BaseEngine.generate", "kind": "method", "signature": "async def generate(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, stop: list[str] | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Input text" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Stop sequences" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "GenerationOutput", "docstring": "Generate a complete response (non-streaming).\n\nArgs:\n prompt: Input text\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n stop: Stop sequences\n **kwargs: Additional model-specific parameters\n\nReturns:\n GenerationOutput with complete text", "summary": "Generate a complete response (non-streaming).", "implementation": "Method `BaseEngine.generate` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 163, "end_line": 186, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L163-L186", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "stream_generate", "qualname": "BaseEngine.stream_generate", "full_name": "vllm_mlx.engine.base.BaseEngine.stream_generate", "kind": "method", "signature": "async def stream_generate(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, stop: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Input text" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Stop sequences" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Stream generation token by token.\n\nArgs:\n prompt: Input text\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n stop: Stop sequences\n **kwargs: Additional model-specific parameters\n\nYields:\n GenerationOutput with incremental text", "summary": "Stream generation token by token.", "implementation": "Method `BaseEngine.stream_generate` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 189, "end_line": 212, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L189-L212", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "chat", "qualname": "BaseEngine.chat", "full_name": "vllm_mlx.engine.base.BaseEngine.chat", "kind": "method", "signature": "async def chat(self, messages: list[dict[str, Any]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list[dict] | None=None, images: list[str] | None=None, videos: list[str] | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "List of chat messages" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional tool definitions" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "GenerationOutput", "docstring": "Chat completion (non-streaming).\n\nArgs:\n messages: List of chat messages\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n tools: Optional tool definitions\n images: Optional image URLs/paths\n videos: Optional video URLs/paths\n **kwargs: Additional model-specific parameters\n\nReturns:\n GenerationOutput with assistant response", "summary": "Chat completion (non-streaming).", "implementation": "Method `BaseEngine.chat` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 215, "end_line": 242, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L215-L242", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "stream_chat", "qualname": "BaseEngine.stream_chat", "full_name": "vllm_mlx.engine.base.BaseEngine.stream_chat", "kind": "method", "signature": "async def stream_chat(self, messages: list[dict[str, Any]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list[dict] | None=None, images: list[str] | None=None, videos: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "List of chat messages" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional tool definitions" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Stream chat completion token by token.\n\nArgs:\n messages: List of chat messages\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n tools: Optional tool definitions\n images: Optional image URLs/paths\n videos: Optional video URLs/paths\n **kwargs: Additional model-specific parameters\n\nYields:\n GenerationOutput with incremental text", "summary": "Stream chat completion token by token.", "implementation": "Method `BaseEngine.stream_chat` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 245, "end_line": 272, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L245-L272", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "BaseEngine.get_stats", "full_name": "vllm_mlx.engine.base.BaseEngine.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict[str, Any]", "parameters": [], "return_annotation": "dict[str, Any]", "docstring": "Get engine statistics. Override in subclasses.", "summary": "Get engine statistics.", "implementation": "Method `BaseEngine.get_stats` returns `{}`.", "documented": true, "public": true, "addressable": true, "line": 274, "end_line": 276, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L274-L276", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{}" ], "awaits": false, "yields": false }, { "name": "get_cache_stats", "qualname": "BaseEngine.get_cache_stats", "full_name": "vllm_mlx.engine.base.BaseEngine.get_cache_stats", "kind": "method", "signature": "def get_cache_stats(self) -> dict[str, Any] | None", "parameters": [], "return_annotation": "dict[str, Any] | None", "docstring": "Get cache statistics. Override in subclasses.", "summary": "Get cache statistics.", "implementation": "Method `BaseEngine.get_cache_stats` returns `None`.", "documented": true, "public": true, "addressable": true, "line": 278, "end_line": 280, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L278-L280", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "clear_runtime_caches", "qualname": "BaseEngine.clear_runtime_caches", "full_name": "vllm_mlx.engine.base.BaseEngine.clear_runtime_caches", "kind": "method", "signature": "def clear_runtime_caches(self) -> dict[str, Any] | None", "parameters": [], "return_annotation": "dict[str, Any] | None", "docstring": "Clear engine-managed runtime caches. Override in subclasses.", "summary": "Clear engine-managed runtime caches.", "implementation": "Method `BaseEngine.clear_runtime_caches` returns `None`.", "documented": true, "public": true, "addressable": true, "line": 282, "end_line": 284, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L282-L284", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "abort_request", "qualname": "BaseEngine.abort_request", "full_name": "vllm_mlx.engine.base.BaseEngine.abort_request", "kind": "method", "signature": "async def abort_request(self, request_id: str) -> bool", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Abort an active or queued request when the engine supports it.", "summary": "Abort an active or queued request when the engine supports it.", "implementation": "Method `BaseEngine.abort_request` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 286, "end_line": 288, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/base.py#L286-L288", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.engine.batched", "path": "vllm_mlx/engine/batched.py", "page_path": "reference/api/vllm_mlx/engine/batched.md", "docstring": "Batched engine for continuous batching with multiple concurrent users.\n\nThis engine wraps AsyncEngineCore to provide continuous batching\nfor better throughput when serving multiple concurrent requests.\n\nFor MLLM models, all requests (text-only and multimodal) are routed through\nthe MLLMScheduler, which handles vision encoding and batched generation via\nMLLMBatchGenerator. MLLM models only initialise the MLLM scheduler (not the\nLLM engine), so text-only requests must also be routed through it.", "summary": "Batched engine for continuous batching with multiple concurrent users.", "line_count": 1231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1-L1231", "members": [ "logger", "_resolve_metal_buffer_cache_limit", "_normalize_tool_call_arguments_for_template", "_extract_media_from_messages", "MLLMModelWrapper", "BatchedEngine" ], "symbols": [ { "name": "_resolve_metal_buffer_cache_limit", "qualname": "_resolve_metal_buffer_cache_limit", "full_name": "vllm_mlx.engine.batched._resolve_metal_buffer_cache_limit", "kind": "function", "signature": "def _resolve_metal_buffer_cache_limit(max_recommended: int, gpu_memory_utilization: float) -> tuple[int, str]", "parameters": [ { "name": "max_recommended", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "gpu_memory_utilization", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[int, str]", "docstring": "Resolve the MLX retained-buffer cache cap for Metal startup.", "summary": "Resolve the MLX retained-buffer cache cap for Metal startup.", "implementation": "Function `_resolve_metal_buffer_cache_limit` calls `os.environ.get`, `int`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 35, "end_line": 58, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L35-L58", "decorators": [], "calls": [ "os.environ.get", "int", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(limit, 'MLX_BUFFER_CACHE_LIMIT')", "(int(max_recommended * gpu_memory_utilization), 'device-scaled')" ], "awaits": false, "yields": false }, { "name": "_normalize_tool_call_arguments_for_template", "qualname": "_normalize_tool_call_arguments_for_template", "full_name": "vllm_mlx.engine.batched._normalize_tool_call_arguments_for_template", "kind": "function", "signature": "def _normalize_tool_call_arguments_for_template(messages: list[dict]) -> list[dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Normalize OpenAI tool-call replay for templates expecting mappings.", "summary": "Normalize OpenAI tool-call replay for templates expecting mappings.", "implementation": "Function `_normalize_tool_call_arguments_for_template` calls `normalize_messages_for_chat_template`; returns `normalize_messages_for_chat_template(messages)`.", "documented": true, "public": false, "addressable": true, "line": 61, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L61-L63", "decorators": [], "calls": [ "normalize_messages_for_chat_template" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "normalize_messages_for_chat_template(messages)" ], "awaits": false, "yields": false }, { "name": "_extract_media_from_messages", "qualname": "_extract_media_from_messages", "full_name": "vllm_mlx.engine.batched._extract_media_from_messages", "kind": "function", "signature": "def _extract_media_from_messages(messages: list[dict[str, Any]]) -> tuple", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple", "docstring": "Extract images, videos, and audio from OpenAI-format messages.\n\nReturns:\n Tuple of (has_media, images_list, videos_list, audios_list)", "summary": "Extract images, videos, and audio from OpenAI-format messages.", "implementation": "Function `_extract_media_from_messages` calls `msg.get`, `isinstance`, `hasattr`, `item.model_dump`; returns `(has_media, images, videos, audios)`.", "documented": true, "public": false, "addressable": true, "line": 66, "end_line": 137, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L66-L137", "decorators": [], "calls": [ "msg.get", "isinstance", "hasattr", "item.model_dump", "item.dict().items", "item.dict", "item.get", "images.append", "img_url.get", "videos.append", "vid_url.get", "audios.append", "audio_url.get", "bool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(has_media, images, videos, audios)" ], "awaits": false, "yields": false }, { "name": "MLLMModelWrapper", "qualname": "MLLMModelWrapper", "full_name": "vllm_mlx.engine.batched.MLLMModelWrapper", "kind": "class", "signature": "class MLLMModelWrapper", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "MLLMModelWrapper", "docstring": "Wrapper for MLLM models to make them compatible with BatchGenerator.\n\nBatchGenerator expects model output to be subscriptable (logits array),\nbut MLLM models return LanguageModelOutput objects. This wrapper extracts\nthe logits from the output.\n\nAlso handles Gemma 3's required pixel_values argument by injecting None\nfor text-only requests.", "summary": "Wrapper for MLLM models to make them compatible with BatchGenerator.", "implementation": "Class `MLLMModelWrapper` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 140, "end_line": 175, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L140-L175", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLLMModelWrapper.__init__", "full_name": "vllm_mlx.engine.batched.MLLMModelWrapper.__init__", "kind": "method", "signature": "def __init__(self, model)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `MLLMModelWrapper.__init__` updates `self._model`, `self._is_gemma3`; calls `hasattr`, `str(getattr(model, 'model_type', '')).lower`, `str`, `getattr`.", "implementation": "Method `MLLMModelWrapper.__init__` updates `self._model`, `self._is_gemma3`; calls `hasattr`, `str(getattr(model, 'model_type', '')).lower`, `str`, `getattr`.", "documented": false, "public": false, "addressable": true, "line": 152, "end_line": 158, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L152-L158", "decorators": [], "calls": [ "hasattr", "str(getattr(model, 'model_type', '')).lower", "str", "getattr" ], "state_reads": [], "state_writes": [ "self._model", "self._is_gemma3" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "MLLMModelWrapper.__call__", "full_name": "vllm_mlx.engine.batched.MLLMModelWrapper.__call__", "kind": "method", "signature": "def __call__(self, *args, **kwargs)", "parameters": [ { "name": "*args", "kind": "variadic positional", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic positional inputs accepted by this callable." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "not annotated", "docstring": "Call the model and extract logits from LanguageModelOutput.", "summary": "Call the model and extract logits from LanguageModelOutput.", "implementation": "Method `MLLMModelWrapper.__call__` calls `self._model`, `hasattr`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 160, "end_line": 171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L160-L171", "decorators": [], "calls": [ "self._model", "hasattr" ], "state_reads": [ "self._is_gemma3", "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "output.logits", "output" ], "awaits": false, "yields": false }, { "name": "__getattr__", "qualname": "MLLMModelWrapper.__getattr__", "full_name": "vllm_mlx.engine.batched.MLLMModelWrapper.__getattr__", "kind": "method", "signature": "def __getattr__(self, name)", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Forward all other attributes to the wrapped model.", "summary": "Forward all other attributes to the wrapped model.", "implementation": "Method `MLLMModelWrapper.__getattr__` calls `getattr`; returns `getattr(self._model, name)`.", "documented": true, "public": false, "addressable": true, "line": 173, "end_line": 175, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L173-L175", "decorators": [], "calls": [ "getattr" ], "state_reads": [ "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "getattr(self._model, name)" ], "awaits": false, "yields": false }, { "name": "BatchedEngine", "qualname": "BatchedEngine", "full_name": "vllm_mlx.engine.batched.BatchedEngine", "kind": "class", "signature": "class BatchedEngine(BaseEngine)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "scheduler_config", "kind": "positional or keyword", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional scheduler configuration" }, { "name": "stream_interval", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Tokens to batch before streaming (1=every token)" }, { "name": "force_mllm", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Force loading as MLLM even if not auto-detected" }, { "name": "gpu_memory_utilization", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Fraction of device memory for Metal allocation limit and emergency threshold (0.0-1.0, default 0.90)" } ], "return_annotation": "BatchedEngine", "docstring": "Batched engine for continuous batching.\n\nThis engine provides better throughput when serving multiple\nconcurrent users by batching requests together.\n\nFor MLLM (multimodal) models, this engine uses MLLMScheduler\nwhich handles images and videos alongside text generation.", "summary": "Batched engine for continuous batching.", "implementation": "Class `BatchedEngine` derives from `BaseEngine` and declares 28 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 178, "end_line": 1231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L178-L1231", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "BatchedEngine.__init__", "full_name": "vllm_mlx.engine.batched.BatchedEngine.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, trust_remote_code: bool=False, scheduler_config: Any | None=None, stream_interval: int=1, force_mllm: bool=False, gpu_memory_utilization: float=0.9)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "scheduler_config", "kind": "positional or keyword", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional scheduler configuration" }, { "name": "stream_interval", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Tokens to batch before streaming (1=every token)" }, { "name": "force_mllm", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Force loading as MLLM even if not auto-detected" }, { "name": "gpu_memory_utilization", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Fraction of device memory for Metal allocation limit and emergency threshold (0.0-1.0, default 0.90)" } ], "return_annotation": "not annotated", "docstring": "Initialize the batched engine.\n\nArgs:\n model_name: HuggingFace model name or local path\n trust_remote_code: Whether to trust remote code\n scheduler_config: Optional scheduler configuration\n stream_interval: Tokens to batch before streaming (1=every token)\n force_mllm: Force loading as MLLM even if not auto-detected\n gpu_memory_utilization: Fraction of device memory for Metal allocation\n limit and emergency threshold (0.0-1.0, default 0.90)", "summary": "Initialize the batched engine.", "implementation": "Method `BatchedEngine.__init__` updates `self._model_name`, `self._created_at`, `self._trust_remote_code`, `self._scheduler_config`; calls `time.time`, `is_mllm_model`.", "documented": true, "public": false, "addressable": true, "line": 189, "end_line": 224, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L189-L224", "decorators": [], "calls": [ "time.time", "is_mllm_model" ], "state_reads": [], "state_writes": [ "self._model_name", "self._created_at", "self._trust_remote_code", "self._scheduler_config", "self._stream_interval", "self._gpu_memory_utilization", "self._is_mllm", "self._model", "self._processor", "self._tokenizer", "self._engine", "self._mllm_scheduler", "self._mllm_instance", "self._loaded" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "model_name", "qualname": "BatchedEngine.model_name", "full_name": "vllm_mlx.engine.batched.BatchedEngine.model_name", "kind": "method", "signature": "def model_name(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Get the model name.", "summary": "Get the model name.", "implementation": "Method `BatchedEngine.model_name` returns `self._model_name`.", "documented": true, "public": true, "addressable": true, "line": 227, "end_line": 229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L227-L229", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._model_name" ], "state_writes": [], "raises": [], "return_expressions": [ "self._model_name" ], "awaits": false, "yields": false }, { "name": "is_mllm", "qualname": "BatchedEngine.is_mllm", "full_name": "vllm_mlx.engine.batched.BatchedEngine.is_mllm", "kind": "method", "signature": "def is_mllm(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if this is a multimodal model.", "summary": "Check if this is a multimodal model.", "implementation": "Method `BatchedEngine.is_mllm` returns `self._is_mllm`.", "documented": true, "public": true, "addressable": true, "line": 232, "end_line": 234, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L232-L234", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._is_mllm" ], "state_writes": [], "raises": [], "return_expressions": [ "self._is_mllm" ], "awaits": false, "yields": false }, { "name": "tokenizer", "qualname": "BatchedEngine.tokenizer", "full_name": "vllm_mlx.engine.batched.BatchedEngine.tokenizer", "kind": "method", "signature": "def tokenizer(self) -> Any", "parameters": [], "return_annotation": "Any", "docstring": "Get the tokenizer.", "summary": "Get the tokenizer.", "implementation": "Method `BatchedEngine.tokenizer` calls `getattr`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 237, "end_line": 241, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L237-L241", "decorators": [ "property" ], "calls": [ "getattr" ], "state_reads": [ "self._is_mllm", "self._processor", "self._tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "getattr(self._processor, 'tokenizer', self._processor)", "self._tokenizer" ], "awaits": false, "yields": false }, { "name": "prepare_for_start", "qualname": "BatchedEngine.prepare_for_start", "full_name": "vllm_mlx.engine.batched.BatchedEngine.prepare_for_start", "kind": "method", "signature": "def prepare_for_start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load heavyweight model state off the serving event loop.", "summary": "Load heavyweight model state off the serving event loop.", "implementation": "Method `BatchedEngine.prepare_for_start` calls `self._prepare_mllm_model`, `self._prepare_llm_model`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 243, "end_line": 251, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L243-L251", "decorators": [], "calls": [ "self._prepare_mllm_model", "self._prepare_llm_model" ], "state_reads": [ "self._model", "self._is_mllm", "self._prepare_mllm_model", "self._prepare_llm_model" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "start", "qualname": "BatchedEngine.start", "full_name": "vllm_mlx.engine.batched.BatchedEngine.start", "kind": "method", "signature": "async def start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the engine (load model if not loaded).", "summary": "Start the engine (load model if not loaded).", "implementation": "Method `BatchedEngine.start` updates `self._loaded`; calls `self._uses_default_prepare_for_start`, `self.prepare_for_start`, `run_blocking_startup_work`, `self._start_mllm`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 253, "end_line": 282, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L253-L282", "decorators": [], "calls": [ "self._uses_default_prepare_for_start", "self.prepare_for_start", "run_blocking_startup_work", "self._start_mllm", "self._start_llm", "logger.info", "cleanup_startup_cancellation" ], "state_reads": [ "self._loaded", "self._model", "self._uses_default_prepare_for_start", "self.prepare_for_start", "self._is_mllm", "self._start_mllm", "self._start_llm", "self._model_name", "self.stop" ], "state_writes": [ "self._loaded" ], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_uses_default_prepare_for_start", "qualname": "BatchedEngine._uses_default_prepare_for_start", "full_name": "vllm_mlx.engine.batched.BatchedEngine._uses_default_prepare_for_start", "kind": "method", "signature": "def _uses_default_prepare_for_start(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return True when prepare_for_start is the class implementation.", "summary": "Return True when prepare_for_start is the class implementation.", "implementation": "Method `BatchedEngine._uses_default_prepare_for_start` calls `getattr`; returns `method is BatchedEngine.prepare_for_start`.", "documented": true, "public": false, "addressable": true, "line": 284, "end_line": 287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L284-L287", "decorators": [], "calls": [ "getattr" ], "state_reads": [ "self.prepare_for_start" ], "state_writes": [], "raises": [], "return_expressions": [ "method is BatchedEngine.prepare_for_start" ], "awaits": false, "yields": false }, { "name": "_prepare_mllm_model", "qualname": "BatchedEngine._prepare_mllm_model", "full_name": "vllm_mlx.engine.batched.BatchedEngine._prepare_mllm_model", "kind": "method", "signature": "def _prepare_mllm_model(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the MLLM model before scheduler startup.", "summary": "Load the MLLM model before scheduler startup.", "implementation": "Method `BatchedEngine._prepare_mllm_model` updates `self._mllm_instance`, `self._model`, `self._processor`; calls `getattr`, `MLXMultimodalLM`, `self._mllm_instance.load`, `mx.metal.is_available`.", "documented": true, "public": false, "addressable": true, "line": 289, "end_line": 334, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L289-L334", "decorators": [], "calls": [ "getattr", "MLXMultimodalLM", "self._mllm_instance.load", "mx.metal.is_available", "mx.device_info", "device_info.get", "int", "_resolve_metal_buffer_cache_limit", "mx.set_memory_limit", "mx.set_cache_limit", "logger.info", "logger.warning", "self._inject_mtp_mllm" ], "state_reads": [ "self._scheduler_config", "self._model_name", "self._trust_remote_code", "self._mllm_instance.load", "self._mllm_instance", "self._mllm_instance.model", "self._mllm_instance.processor", "self._gpu_memory_utilization", "self._scheduler_config.enable_mtp", "self._inject_mtp_mllm" ], "state_writes": [ "self._mllm_instance", "self._model", "self._processor" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_start_mllm", "qualname": "BatchedEngine._start_mllm", "full_name": "vllm_mlx.engine.batched.BatchedEngine._start_mllm", "kind": "method", "signature": "async def _start_mllm(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the MLLM engine with MLLMScheduler (continuous batching).", "summary": "Start the MLLM engine with MLLMScheduler (continuous batching).", "implementation": "Method `BatchedEngine._start_mllm` updates `self._mllm_scheduler`; calls `self._prepare_mllm_model`, `hasattr`, `getattr`, `MLLMSchedulerConfig`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 336, "end_line": 429, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L336-L429", "decorators": [], "calls": [ "self._prepare_mllm_model", "hasattr", "getattr", "MLLMSchedulerConfig", "MLLMScheduler", "self._mllm_scheduler.start", "logger.info" ], "state_reads": [ "self._model", "self._processor", "self._prepare_mllm_model", "self._scheduler_config", "self._scheduler_config.max_num_seqs", "self._scheduler_config.enable_mtp", "self._mllm_scheduler.start", "self._mllm_scheduler" ], "state_writes": [ "self._mllm_scheduler" ], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_inject_mtp_mllm", "qualname": "BatchedEngine._inject_mtp_mllm", "full_name": "vllm_mlx.engine.batched.BatchedEngine._inject_mtp_mllm", "kind": "method", "signature": "def _inject_mtp_mllm(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Inject MTP weights into the MLLM model's language_model.", "summary": "Inject MTP weights into the MLLM model's language_model.", "implementation": "Method `BatchedEngine._inject_mtp_mllm` calls `Path`, `_download`, `config_path.exists`, `logger.warning`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 431, "end_line": 477, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L431-L477", "decorators": [], "calls": [ "Path", "_download", "config_path.exists", "logger.warning", "open", "json.load", "config.get", "text_config.get", "logger.info", "hasattr", "getattr", "inject_mtp_support" ], "state_reads": [ "self._model", "self._model_name" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_prepare_llm_model", "qualname": "BatchedEngine._prepare_llm_model", "full_name": "vllm_mlx.engine.batched.BatchedEngine._prepare_llm_model", "kind": "method", "signature": "def _prepare_llm_model(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the LLM model/tokenizer before engine loop startup.", "summary": "Load the LLM model/tokenizer before engine loop startup.", "implementation": "Method `BatchedEngine._prepare_llm_model` updates `self._model`, `self._tokenizer`; calls `self._model_name.lower`, `load_model_with_fallback`, `validate_mtp_support`, `validate_35`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 479, "end_line": 511, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L479-L511", "decorators": [], "calls": [ "self._model_name.lower", "load_model_with_fallback", "validate_mtp_support", "validate_35", "logger.info", "logger.warning", "self._configure_metal_memory_limits" ], "state_reads": [ "self._model", "self._tokenizer", "self._trust_remote_code", "self._model_name.lower", "self._model_name", "self._scheduler_config", "self._scheduler_config.enable_mtp", "self._configure_metal_memory_limits" ], "state_writes": [ "self._model", "self._tokenizer" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_configure_metal_memory_limits", "qualname": "BatchedEngine._configure_metal_memory_limits", "full_name": "vllm_mlx.engine.batched.BatchedEngine._configure_metal_memory_limits", "kind": "method", "signature": "def _configure_metal_memory_limits(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Make MLX allocation failures graceful during startup.", "summary": "Make MLX allocation failures graceful during startup.", "implementation": "Method `BatchedEngine._configure_metal_memory_limits` calls `mx.metal.is_available`, `mx.device_info`, `device_info.get`, `int`.", "documented": true, "public": false, "addressable": true, "line": 513, "end_line": 541, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L513-L541", "decorators": [], "calls": [ "mx.metal.is_available", "mx.device_info", "device_info.get", "int", "_resolve_metal_buffer_cache_limit", "mx.set_memory_limit", "mx.set_cache_limit", "logger.info", "logger.warning" ], "state_reads": [ "self._gpu_memory_utilization" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_start_llm", "qualname": "BatchedEngine._start_llm", "full_name": "vllm_mlx.engine.batched.BatchedEngine._start_llm", "kind": "method", "signature": "async def _start_llm(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the LLM engine with AsyncEngineCore.", "summary": "Start the LLM engine with AsyncEngineCore.", "implementation": "Method `BatchedEngine._start_llm` updates `self._engine`; calls `self._prepare_llm_model`, `validate_mtp_support`, `logger.info`, `logger.warning`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 543, "end_line": 579, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L543-L579", "decorators": [], "calls": [ "self._prepare_llm_model", "validate_mtp_support", "logger.info", "logger.warning", "SchedulerConfig", "EngineConfig", "AsyncEngineCore", "self._engine.engine.start" ], "state_reads": [ "self._model", "self._tokenizer", "self._prepare_llm_model", "self._scheduler_config", "self._scheduler_config.enable_mtp", "self._model_name", "self._stream_interval", "self._gpu_memory_utilization", "self._engine.engine.start", "self._engine.engine", "self._engine" ], "state_writes": [ "self._engine" ], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "stop", "qualname": "BatchedEngine.stop", "full_name": "vllm_mlx.engine.batched.BatchedEngine.stop", "kind": "method", "signature": "async def stop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop the engine and cleanup resources.", "summary": "Stop the engine and cleanup resources.", "implementation": "Method `BatchedEngine.stop` updates `self._mllm_scheduler`, `self._engine`, `self._model`, `self._tokenizer`; calls `self._mllm_scheduler.stop`, `self._engine.stop`, `self._engine.engine.close`, `logger.info`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 581, "end_line": 597, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L581-L597", "decorators": [], "calls": [ "self._mllm_scheduler.stop", "self._engine.stop", "self._engine.engine.close", "logger.info" ], "state_reads": [ "self._mllm_scheduler", "self._mllm_scheduler.stop", "self._engine", "self._engine.stop", "self._engine.engine.close", "self._engine.engine" ], "state_writes": [ "self._mllm_scheduler", "self._engine", "self._model", "self._tokenizer", "self._processor", "self._mllm_instance", "self._loaded" ], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_apply_chat_template", "qualname": "BatchedEngine._apply_chat_template", "full_name": "vllm_mlx.engine.batched.BatchedEngine._apply_chat_template", "kind": "method", "signature": "def _apply_chat_template(self, messages: list[dict[str, Any]], tools: list[dict] | None=None, num_images: int=0, num_audios: int=0, chat_template_kwargs: dict[str, Any] | None=None, enable_thinking: bool | None=None) -> str", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "num_images", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." }, { "name": "num_audios", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." }, { "name": "chat_template_kwargs", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "enable_thinking", "kind": "positional or keyword", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "str", "docstring": "Apply chat template to messages.\n\nUses the processor's (or tokenizer's) apply_chat_template with the\nfull message list so that system prompts and conversation history\nare preserved. The previous implementation extracted only the last\nuser message text via mlx_vlm.prompt_utils.apply_chat_template,\nwhich dropped system prompts and all prior turns.", "summary": "Apply chat template to messages.", "implementation": "Method `BatchedEngine._apply_chat_template` calls `_normalize_tool_call_arguments_for_template`, `hasattr`, `self._prepare_mllm_messages`, `self._model_name.lower`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 599, "end_line": 687, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L599-L687", "decorators": [], "calls": [ "_normalize_tool_call_arguments_for_template", "hasattr", "self._prepare_mllm_messages", "self._model_name.lower", "template_kwargs.update", "template_applicator.apply_chat_template", "str", "tokenizer_applicator.apply_chat_template", "logger.debug", "(chat_template_kwargs or {}).keys", "template_kwargs.pop", "'\\n'.join" ], "state_reads": [ "self._is_mllm", "self._processor", "self.tokenizer", "self._prepare_mllm_messages", "self._model_name.lower", "self._model_name" ], "state_writes": [], "raises": [], "return_expressions": [ "template_applicator.apply_chat_template(messages, **template_kwargs)", "tokenizer_applicator.apply_chat_template(messages, **template_kwargs)", "prompt + '\\nassistant:'" ], "awaits": false, "yields": false }, { "name": "_prepare_mllm_messages", "qualname": "BatchedEngine._prepare_mllm_messages", "full_name": "vllm_mlx.engine.batched.BatchedEngine._prepare_mllm_messages", "kind": "method", "signature": "def _prepare_mllm_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "List of chat messages in OpenAI format. Each message is a dict with at least ``role`` and ``content`` keys." } ], "return_annotation": "list[dict[str, Any]]", "docstring": "Convert OpenAI-style multimodal content to HuggingFace format.\n\nThe OpenAI API uses ``{\"type\": \"image_url\", \"image_url\": {\"url\": ...}}``\nand ``{\"type\": \"audio_url\", \"audio_url\": {\"url\": ...}}`` while\nHuggingFace processors expect ``{\"type\": \"image\"}`` / ``{\"type\": \"audio\"}``.\n\nArgs:\n messages: List of chat messages in OpenAI format. Each message is a\n dict with at least ``role`` and ``content`` keys.\n\nReturns:\n A new list of messages with ``image_url`` / ``audio_url`` parts\n replaced by ``{\"type\": \"image\"}`` / ``{\"type\": \"audio\"}`` entries\n for the HuggingFace processor.", "summary": "Convert OpenAI-style multimodal content to HuggingFace format.", "implementation": "Method `BatchedEngine._prepare_mllm_messages` calls `isinstance`, `msg.get`, `part.get`, `new_content.append`; returns `prepared`.", "documented": true, "public": false, "addressable": true, "line": 690, "end_line": 726, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L690-L726", "decorators": [ "staticmethod" ], "calls": [ "isinstance", "msg.get", "part.get", "new_content.append", "prepared.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "prepared" ], "awaits": false, "yields": false }, { "name": "generate", "qualname": "BatchedEngine.generate", "full_name": "vllm_mlx.engine.batched.BatchedEngine.generate", "kind": "method", "signature": "async def generate(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, stop: list[str] | None=None, images: list[str] | None=None, videos: list[str] | None=None, audio: list[str] | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Input text" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Stop sequences" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths (for MLLM)" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths (for MLLM)" }, { "name": "audio", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional audio URLs/paths (for MLLM)" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "GenerationOutput", "docstring": "Generate a complete response (non-streaming).\n\nArgs:\n prompt: Input text\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n stop: Stop sequences\n images: Optional image URLs/paths (for MLLM)\n videos: Optional video URLs/paths (for MLLM)\n audio: Optional audio URLs/paths (for MLLM)\n **kwargs: Additional model-specific parameters\n\nReturns:\n GenerationOutput with complete text", "summary": "Generate a complete response (non-streaming).", "implementation": "Method `BatchedEngine.generate` calls `self.start`, `self._mllm_scheduler.generate`, `kwargs.pop`, `GenerationOutput`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 728, "end_line": 817, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L728-L817", "decorators": [], "calls": [ "self.start", "self._mllm_scheduler.generate", "kwargs.pop", "GenerationOutput", "clean_output_text", "SamplingParams", "self._engine.generate" ], "state_reads": [ "self._loaded", "self.start", "self._is_mllm", "self._mllm_scheduler", "self._mllm_scheduler.generate", "self._engine.generate", "self._engine" ], "state_writes": [], "raises": [], "return_expressions": [ "GenerationOutput(text=clean_output_text(output.output_text), tokens=output.output_token_ids, prompt_tokens=output.promp…", "GenerationOutput(text=text, tokens=output.output_token_ids, prompt_tokens=output.prompt_tokens, completion_tokens=outpu…" ], "awaits": true, "yields": false }, { "name": "stream_generate", "qualname": "BatchedEngine.stream_generate", "full_name": "vllm_mlx.engine.batched.BatchedEngine.stream_generate", "kind": "method", "signature": "async def stream_generate(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, stop: list[str] | None=None, images: list[str] | None=None, videos: list[str] | None=None, audio: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Input text" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Stop sequences" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths (for MLLM)" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths (for MLLM)" }, { "name": "audio", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional audio URLs/paths (for MLLM)" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Stream generation token by token.\n\nArgs:\n prompt: Input text\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n stop: Stop sequences\n images: Optional image URLs/paths (for MLLM)\n videos: Optional video URLs/paths (for MLLM)\n audio: Optional audio URLs/paths (for MLLM)\n **kwargs: Additional model-specific parameters\n\nYields:\n GenerationOutput with incremental text", "summary": "Stream generation token by token.", "implementation": "Method `BatchedEngine.stream_generate` calls `self.start`, `self._mllm_scheduler.add_request_async`, `kwargs.pop`, `self._mllm_scheduler.stream_outputs`; awaits asynchronous work; yields values incrementally; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 819, "end_line": 913, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L819-L913", "decorators": [], "calls": [ "self.start", "self._mllm_scheduler.add_request_async", "kwargs.pop", "self._mllm_scheduler.stream_outputs", "GenerationOutput", "clean_output_text", "SamplingParams", "self._engine.add_request", "self._engine.stream_outputs" ], "state_reads": [ "self._loaded", "self.start", "self._is_mllm", "self._mllm_scheduler", "self._mllm_scheduler.add_request_async", "self._mllm_scheduler.stream_outputs", "self._engine.add_request", "self._engine", "self._engine.stream_outputs" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": true }, { "name": "chat", "qualname": "BatchedEngine.chat", "full_name": "vllm_mlx.engine.batched.BatchedEngine.chat", "kind": "method", "signature": "async def chat(self, messages: list[dict[str, Any]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list[dict] | None=None, images: list[str] | None=None, videos: list[str] | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "List of chat messages (OpenAI format)" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional tool definitions" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "GenerationOutput", "docstring": "Chat completion (non-streaming).\n\nFor MLLM models, all requests (including text-only) are routed through\nthe MLLMScheduler for vision-aware batched generation.\nFor non-MLLM models, uses the LLM engine with BatchGenerator.\n\nArgs:\n messages: List of chat messages (OpenAI format)\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n tools: Optional tool definitions\n images: Optional image URLs/paths\n videos: Optional video URLs/paths\n **kwargs: Additional model-specific parameters\n\nReturns:\n GenerationOutput with assistant response", "summary": "Chat completion (non-streaming).", "implementation": "Method `BatchedEngine.chat` calls `self.start`, `extract_multimodal_content`, `convert_tools_for_template`, `dict`; awaits asynchronous work; returns `await self.generate(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, images=all_images if al…`.", "documented": true, "public": true, "addressable": true, "line": 915, "end_line": 984, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L915-L984", "decorators": [], "calls": [ "self.start", "extract_multimodal_content", "convert_tools_for_template", "dict", "kwargs.pop", "self._apply_chat_template", "len", "self.generate" ], "state_reads": [ "self._loaded", "self.start", "self._apply_chat_template", "self.generate" ], "state_writes": [], "raises": [], "return_expressions": [ "await self.generate(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, images=all_images if al…" ], "awaits": true, "yields": false }, { "name": "_compute_prefix_boundary", "qualname": "BatchedEngine._compute_prefix_boundary", "full_name": "vllm_mlx.engine.batched.BatchedEngine._compute_prefix_boundary", "kind": "method", "signature": "def _compute_prefix_boundary(self, messages: list[dict[str, Any]], tools: list[dict] | None=None, chat_template_kwargs: dict[str, Any] | None=None) -> int", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "chat_template_kwargs", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "int", "docstring": "Compute token count for the shared prefix across message variations.\n\nUses a two-tokenization approach: tokenize the full prompt twice\n(once as-is, once with the last user message replaced by a dummy)\nand find the longest common prefix (LCP). This gives the exact\nboundary where different user suffixes diverge, avoiding template\ndiscrepancies (e.g. Qwen3 markers on last assistant).", "summary": "Compute token count for the shared prefix across message variations.", "implementation": "Method `BatchedEngine._compute_prefix_boundary` calls `range`, `len`, `messages[i].get`, `convert_tools_for_template`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 986, "end_line": 1046, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L986-L1046", "decorators": [], "calls": [ "range", "len", "messages[i].get", "convert_tools_for_template", "self._apply_chat_template", "list", "hasattr", "tokenizer.encode", "min" ], "state_reads": [ "self._apply_chat_template", "self.tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "0", "lcp" ], "awaits": false, "yields": false }, { "name": "stream_chat", "qualname": "BatchedEngine.stream_chat", "full_name": "vllm_mlx.engine.batched.BatchedEngine.stream_chat", "kind": "method", "signature": "async def stream_chat(self, messages: list[dict[str, Any]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list[dict] | None=None, images: list[str] | None=None, videos: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "List of chat messages (OpenAI format)" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional tool definitions" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Stream chat completion token by token.\n\nFor MLLM models, all requests (including text-only) are streamed through\nthe MLLMScheduler for vision-aware batched generation.\nFor non-MLLM models, uses the LLM engine with BatchGenerator.\n\nArgs:\n messages: List of chat messages (OpenAI format)\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n tools: Optional tool definitions\n images: Optional image URLs/paths\n videos: Optional video URLs/paths\n **kwargs: Additional model-specific parameters\n\nYields:\n GenerationOutput with incremental text", "summary": "Stream chat completion token by token.", "implementation": "Method `BatchedEngine.stream_chat` calls `self.start`, `extract_multimodal_content`, `convert_tools_for_template`, `dict`; awaits asynchronous work; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 1048, "end_line": 1127, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1048-L1127", "decorators": [], "calls": [ "self.start", "extract_multimodal_content", "convert_tools_for_template", "dict", "kwargs.pop", "self._apply_chat_template", "len", "self._compute_prefix_boundary", "self.stream_generate" ], "state_reads": [ "self._loaded", "self.start", "self._apply_chat_template", "self._compute_prefix_boundary", "self.stream_generate" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": true }, { "name": "get_stats", "qualname": "BatchedEngine.get_stats", "full_name": "vllm_mlx.engine.batched.BatchedEngine.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict[str, Any]", "parameters": [], "return_annotation": "dict[str, Any]", "docstring": "Get engine statistics.", "summary": "Get engine statistics.", "implementation": "Method `BatchedEngine.get_stats` calls `time.time`, `self._mllm_scheduler.get_stats`, `stats.update`, `self._engine.get_stats`; returns `stats`.", "documented": true, "public": true, "addressable": true, "line": 1129, "end_line": 1169, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1129-L1169", "decorators": [], "calls": [ "time.time", "self._mllm_scheduler.get_stats", "stats.update", "self._engine.get_stats" ], "state_reads": [ "self._model_name", "self._created_at", "self._is_mllm", "self._loaded", "self._stream_interval", "self._mllm_scheduler", "self._mllm_scheduler.get_stats", "self._engine", "self._engine.get_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "stats" ], "awaits": false, "yields": false }, { "name": "get_cache_stats", "qualname": "BatchedEngine.get_cache_stats", "full_name": "vllm_mlx.engine.batched.BatchedEngine.get_cache_stats", "kind": "method", "signature": "def get_cache_stats(self) -> dict[str, Any] | None", "parameters": [], "return_annotation": "dict[str, Any] | None", "docstring": "Get cache statistics.", "summary": "Get cache statistics.", "implementation": "Method `BatchedEngine.get_cache_stats` calls `self._mllm_scheduler.batch_generator.get_prefix_cache_stats`, `self._mllm_scheduler.batch_generator.get_vision_cache_stats`, `self._engine.get_cache_stats`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1171, "end_line": 1180, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1171-L1180", "decorators": [], "calls": [ "self._mllm_scheduler.batch_generator.get_prefix_cache_stats", "self._mllm_scheduler.batch_generator.get_vision_cache_stats", "self._engine.get_cache_stats" ], "state_reads": [ "self._mllm_scheduler", "self._mllm_scheduler.batch_generator", "self._mllm_scheduler.batch_generator.get_prefix_cache_stats", "self._mllm_scheduler.batch_generator.get_vision_cache_stats", "self._engine", "self._engine.get_cache_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "{'prefix_cache': self._mllm_scheduler.batch_generator.get_prefix_cache_stats(), 'vision_embedding_cache': self._mllm_sc…", "self._engine.get_cache_stats()", "None" ], "awaits": false, "yields": false }, { "name": "clear_runtime_caches", "qualname": "BatchedEngine.clear_runtime_caches", "full_name": "vllm_mlx.engine.batched.BatchedEngine.clear_runtime_caches", "kind": "method", "signature": "def clear_runtime_caches(self) -> dict[str, Any] | None", "parameters": [], "return_annotation": "dict[str, Any] | None", "docstring": "Clear engine-managed runtime caches.", "summary": "Clear engine-managed runtime caches.", "implementation": "Method `BatchedEngine.clear_runtime_caches` calls `self._mllm_scheduler.clear_runtime_caches`, `self._engine.clear_runtime_caches`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1182, "end_line": 1188, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1182-L1188", "decorators": [], "calls": [ "self._mllm_scheduler.clear_runtime_caches", "self._engine.clear_runtime_caches" ], "state_reads": [ "self._mllm_scheduler", "self._mllm_scheduler.clear_runtime_caches", "self._engine", "self._engine.clear_runtime_caches" ], "state_writes": [], "raises": [], "return_expressions": [ "self._mllm_scheduler.clear_runtime_caches()", "self._engine.clear_runtime_caches()", "None" ], "awaits": false, "yields": false }, { "name": "abort_request", "qualname": "BatchedEngine.abort_request", "full_name": "vllm_mlx.engine.batched.BatchedEngine.abort_request", "kind": "method", "signature": "async def abort_request(self, request_id: str) -> bool", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Abort an active or queued batched request by request ID.", "summary": "Abort an active or queued batched request by request ID.", "implementation": "Method `BatchedEngine.abort_request` calls `self._mllm_scheduler.abort_request`, `hasattr`, `self._engine.abort_request`, `inspect.isawaitable`; awaits asynchronous work; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1190, "end_line": 1199, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1190-L1199", "decorators": [], "calls": [ "self._mllm_scheduler.abort_request", "hasattr", "self._engine.abort_request", "inspect.isawaitable" ], "state_reads": [ "self._mllm_scheduler", "self._mllm_scheduler.abort_request", "self._engine", "self._engine.abort_request" ], "state_writes": [], "raises": [], "return_expressions": [ "self._mllm_scheduler.abort_request(request_id)", "await result", "result", "False" ], "awaits": true, "yields": false }, { "name": "save_cache_to_disk", "qualname": "BatchedEngine.save_cache_to_disk", "full_name": "vllm_mlx.engine.batched.BatchedEngine.save_cache_to_disk", "kind": "method", "signature": "def save_cache_to_disk(self, cache_dir: str) -> bool", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Save prefix cache to disk for persistence across restarts.", "summary": "Save prefix cache to disk for persistence across restarts.", "implementation": "Method `BatchedEngine.save_cache_to_disk` calls `pc.save_to_disk`, `self._engine.save_cache_to_disk`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1201, "end_line": 1209, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1201-L1209", "decorators": [], "calls": [ "pc.save_to_disk", "self._engine.save_cache_to_disk" ], "state_reads": [ "self._mllm_scheduler", "self._mllm_scheduler.batch_generator", "self._mllm_scheduler.batch_generator.prefix_cache", "self._engine", "self._engine.save_cache_to_disk" ], "state_writes": [], "raises": [], "return_expressions": [ "pc.save_to_disk(cache_dir)", "self._engine.save_cache_to_disk(cache_dir)", "False" ], "awaits": false, "yields": false }, { "name": "load_cache_from_disk", "qualname": "BatchedEngine.load_cache_from_disk", "full_name": "vllm_mlx.engine.batched.BatchedEngine.load_cache_from_disk", "kind": "method", "signature": "def load_cache_from_disk(self, cache_dir: str) -> int", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Load prefix cache from disk. Returns number of entries loaded.", "summary": "Load prefix cache from disk.", "implementation": "Method `BatchedEngine.load_cache_from_disk` calls `self._mllm_scheduler._ensure_batch_generator`, `pc.load_from_disk`, `self._engine.load_cache_from_disk`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1211, "end_line": 1220, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1211-L1220", "decorators": [], "calls": [ "self._mllm_scheduler._ensure_batch_generator", "pc.load_from_disk", "self._engine.load_cache_from_disk" ], "state_reads": [ "self._mllm_scheduler", "self._mllm_scheduler._ensure_batch_generator", "self._mllm_scheduler.batch_generator.prefix_cache", "self._mllm_scheduler.batch_generator", "self._engine", "self._engine.load_cache_from_disk" ], "state_writes": [], "raises": [], "return_expressions": [ "pc.load_from_disk(cache_dir)", "self._engine.load_cache_from_disk(cache_dir)", "0" ], "awaits": false, "yields": false }, { "name": "clear_prefix_cache", "qualname": "BatchedEngine.clear_prefix_cache", "full_name": "vllm_mlx.engine.batched.BatchedEngine.clear_prefix_cache", "kind": "method", "signature": "def clear_prefix_cache(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear the in-memory prefix cache. Used by bench-serve for clean\ncold-start measurements between configurations.", "summary": "Clear the in-memory prefix cache.", "implementation": "Method `BatchedEngine.clear_prefix_cache` calls `hasattr`, `pc.clear`, `self._engine.clear_prefix_cache`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 1222, "end_line": 1231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/batched.py#L1222-L1231", "decorators": [], "calls": [ "hasattr", "pc.clear", "self._engine.clear_prefix_cache" ], "state_reads": [ "self._mllm_scheduler", "self._mllm_scheduler.batch_generator", "self._mllm_scheduler.batch_generator.prefix_cache", "self._engine", "self._engine.clear_prefix_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.engine.chat_template_safety", "path": "vllm_mlx/engine/chat_template_safety.py", "page_path": "reference/api/vllm_mlx/engine/chat_template_safety.md", "docstring": "Safety normalization for messages before Jinja chat-template rendering.", "summary": "Safety normalization for messages before Jinja chat-template rendering.", "line_count": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/chat_template_safety.py#L1-L90", "members": [ "_close_dangling_think_before_tool_call", "_message_to_dict", "normalize_messages_for_chat_template" ], "symbols": [ { "name": "_close_dangling_think_before_tool_call", "qualname": "_close_dangling_think_before_tool_call", "full_name": "vllm_mlx.engine.chat_template_safety._close_dangling_think_before_tool_call", "kind": "function", "signature": "def _close_dangling_think_before_tool_call(content: str) -> str", "parameters": [ { "name": "content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Keep raw tool XML out of an unterminated ```` section.\n\nQwen 3.6 can produce assistant history where ```` is opened and a\nraw ```` follows before ````. Rendering that history as-is\nconditions the next turn as though the tool call is still reasoning. Close\nthe dangling thinking span immediately before the first tool call.\n\nThis mirrors the template-side repair described by Cheuk-Yiu Chan:\nhttps://allanchan339.github.io/bug-fixes/2026/05/02/Qwen36-27B-updated-jinja.html", "summary": "Keep raw tool XML out of an unterminated ```` section.", "implementation": "Function `_close_dangling_think_before_tool_call` calls `content.rfind`, `content.find`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 8, "end_line": 29, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/chat_template_safety.py#L8-L29", "decorators": [], "calls": [ "content.rfind", "content.find" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "content", "content[:tool_pos] + '' + content[tool_pos:]", "content + ''" ], "awaits": false, "yields": false }, { "name": "_message_to_dict", "qualname": "_message_to_dict", "full_name": "vllm_mlx.engine.chat_template_safety._message_to_dict", "kind": "function", "signature": "def _message_to_dict(message: Any) -> dict[str, Any] | Any", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any] | Any", "docstring": "Convert OpenAI message model objects without stringifying them.", "summary": "Convert OpenAI message model objects without stringifying them.", "implementation": "Function `_message_to_dict` calls `isinstance`, `dict`, `getattr`, `callable`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 32, "end_line": 46, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/chat_template_safety.py#L32-L46", "decorators": [], "calls": [ "isinstance", "dict", "getattr", "callable", "model_dump(exclude_none=True).items", "model_dump", "legacy_dict().items", "legacy_dict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "dict(message)", "{key: value for key, value in model_dump(exclude_none=True).items() if value is not None}", "{k: v for k, v in legacy_dict().items() if v is not None}", "message" ], "awaits": false, "yields": false }, { "name": "normalize_messages_for_chat_template", "qualname": "normalize_messages_for_chat_template", "full_name": "vllm_mlx.engine.chat_template_safety.normalize_messages_for_chat_template", "kind": "function", "signature": "def normalize_messages_for_chat_template(messages: list[Any]) -> list[dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Return a JSON-safe copy of messages for chat-template rendering.\n\nNormalizations:\n- close dangling ```` spans before raw ```` XML in\n assistant content\n- convert OpenAI tool-call argument JSON strings to mappings for templates\n that iterate argument keys", "summary": "Return a JSON-safe copy of messages for chat-template rendering.", "implementation": "Function `normalize_messages_for_chat_template` calls `json.loads`, `json.dumps`, `_message_to_dict`, `isinstance`; returns `normalized`.", "documented": true, "public": true, "addressable": true, "line": 49, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/chat_template_safety.py#L49-L90", "decorators": [], "calls": [ "json.loads", "json.dumps", "_message_to_dict", "isinstance", "message.get", "_close_dangling_think_before_tool_call", "tool_call.get", "function.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "normalized" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.engine.simple", "path": "vllm_mlx/engine/simple.py", "page_path": "reference/api/vllm_mlx/engine/simple.md", "docstring": "Simple engine for maximum single-user throughput.\n\nThis engine wraps mlx-lm directly with zero overhead for optimal\nperformance when serving a single user at a time.", "summary": "Simple engine for maximum single-user throughput.", "line_count": 2912, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1-L2912", "members": [ "_in_tracker", "logger", "_bind_worker_generation_streams", "_seed_logits_processors", "_sample_with_processors", "_processors_can_retire", "_processors_retired", "_SpecPrefillCancelled", "SimpleEngine" ], "symbols": [ { "name": "_bind_worker_generation_streams", "qualname": "_bind_worker_generation_streams", "full_name": "vllm_mlx.engine.simple._bind_worker_generation_streams", "kind": "function", "signature": "def _bind_worker_generation_streams() -> None", "parameters": [], "return_annotation": "None", "docstring": "Rebind mlx generation streams inside the current worker thread.", "summary": "Rebind mlx generation streams inside the current worker thread.", "implementation": "Function `_bind_worker_generation_streams` calls `bind_generation_streams`.", "documented": true, "public": false, "addressable": true, "line": 48, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L48-L50", "decorators": [], "calls": [ "bind_generation_streams" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_seed_logits_processors", "qualname": "_seed_logits_processors", "full_name": "vllm_mlx.engine.simple._seed_logits_processors", "kind": "function", "signature": "def _seed_logits_processors(seed_tokens: mx.array | None, processors: list[Any] | None) -> list[Any] | None", "parameters": [ { "name": "seed_tokens", "kind": "positional or keyword", "annotation": "mx.array | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "processors", "kind": "positional or keyword", "annotation": "list[Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[Any] | None", "docstring": "Wrap logits processors so continuation decode sees the full prompt.", "summary": "Wrap logits processors so continuation decode sees the full prompt.", "implementation": "Function `_seed_logits_processors` calls `list`, `_wrap`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 53, "end_line": 77, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L53-L77", "decorators": [], "calls": [ "list", "_wrap" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "list(processors)", "[_wrap(processor) for processor in processors]" ], "awaits": false, "yields": false }, { "name": "_wrap", "qualname": "_seed_logits_processors._wrap", "full_name": "vllm_mlx.engine.simple._seed_logits_processors._wrap", "kind": "nested function", "signature": "def _wrap(processor)", "parameters": [ { "name": "processor", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_seed_logits_processors._wrap` returns `_seeded`.", "implementation": "Nested Function `_seed_logits_processors._wrap` returns `_seeded`.", "documented": false, "public": false, "addressable": false, "line": 63, "end_line": 75, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L63-L75", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_seeded" ], "awaits": false, "yields": false }, { "name": "_seeded", "qualname": "_seed_logits_processors._wrap._seeded", "full_name": "vllm_mlx.engine.simple._seed_logits_processors._wrap._seeded", "kind": "nested function", "signature": "def _seeded(tokens, logits)", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_seed_logits_processors._wrap._seeded` calls `isinstance`, `mx.array`, `mx.concatenate`, `processor`; returns `processor(merged, logits)`.", "implementation": "Nested Function `_seed_logits_processors._wrap._seeded` calls `isinstance`, `mx.array`, `mx.concatenate`, `processor`; returns `processor(merged, logits)`.", "documented": false, "public": false, "addressable": false, "line": 64, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L64-L73", "decorators": [], "calls": [ "isinstance", "mx.array", "mx.concatenate", "processor" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "processor(merged, logits)" ], "awaits": false, "yields": false }, { "name": "_sample_with_processors", "qualname": "_sample_with_processors", "full_name": "vllm_mlx.engine.simple._sample_with_processors", "kind": "function", "signature": "def _sample_with_processors(tokens: mx.array | None, logits: mx.array, sampler: Any, logits_processors: list[Any] | None) -> tuple[mx.array, mx.array]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "mx.array | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "sampler", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits_processors", "kind": "positional or keyword", "annotation": "list[Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[mx.array, mx.array]", "docstring": "Sample a token while honoring any active logits processors.", "summary": "Sample a token while honoring any active logits processors.", "implementation": "Function `_sample_with_processors` calls `processor`, `logits.squeeze`, `mx.logsumexp`, `sampler`; returns `(tok, logprobs)`.", "documented": true, "public": false, "addressable": true, "line": 80, "end_line": 97, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L80-L97", "decorators": [], "calls": [ "processor", "logits.squeeze", "mx.logsumexp", "sampler" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(tok, logprobs)" ], "awaits": false, "yields": false }, { "name": "_processors_can_retire", "qualname": "_processors_can_retire", "full_name": "vllm_mlx.engine.simple._processors_can_retire", "kind": "function", "signature": "def _processors_can_retire(processors: list[Any] | None) -> bool", "parameters": [ { "name": "processors", "kind": "positional or keyword", "annotation": "list[Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "True when any processor advertises a retire-to-content transition.", "summary": "True when any processor advertises a retire-to-content transition.", "implementation": "Function `_processors_can_retire` calls `os.getenv`, `bool`, `any`, `isinstance`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 100, "end_line": 106, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L100-L106", "decorators": [], "calls": [ "os.getenv", "bool", "any", "isinstance", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "bool(processors) and any((isinstance(getattr(p, 'is_retired', None), bool) for p in processors))" ], "awaits": false, "yields": false }, { "name": "_processors_retired", "qualname": "_processors_retired", "full_name": "vllm_mlx.engine.simple._processors_retired", "kind": "function", "signature": "def _processors_retired(processors: list[Any] | None) -> bool", "parameters": [ { "name": "processors", "kind": "positional or keyword", "annotation": "list[Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "True when any retire-capable processor has entered its retired state.", "summary": "True when any retire-capable processor has entered its retired state.", "implementation": "Function `_processors_retired` calls `os.getenv`, `bool`, `any`, `getattr`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 109, "end_line": 115, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L109-L115", "decorators": [], "calls": [ "os.getenv", "bool", "any", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "bool(processors) and any((getattr(p, 'is_retired', False) is True for p in processors))" ], "awaits": false, "yields": false }, { "name": "_SpecPrefillCancelled", "qualname": "_SpecPrefillCancelled", "full_name": "vllm_mlx.engine.simple._SpecPrefillCancelled", "kind": "class", "signature": "class _SpecPrefillCancelled(Exception)", "parameters": [], "return_annotation": "_SpecPrefillCancelled", "docstring": "Cooperative cancellation sentinel for blocking SpecPrefill workers.", "summary": "Cooperative cancellation sentinel for blocking SpecPrefill workers.", "implementation": "Class `_SpecPrefillCancelled` derives from `Exception` and declares 0 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 118, "end_line": 119, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L118-L119", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "SimpleEngine", "qualname": "SimpleEngine", "full_name": "vllm_mlx.engine.simple.SimpleEngine", "kind": "class", "signature": "class SimpleEngine(BaseEngine)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "enable_cache", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Enable VLM cache for multimodal models" }, { "name": "force_mllm", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Force loading as MLLM even if not auto-detected" }, { "name": "mtp", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable native MTP speculative decoding (model must have MTP head)" }, { "name": "mtp_num_draft_tokens", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Draft tokens per speculative MTP step" }, { "name": "prefill_step_size", "kind": "positional or keyword", "annotation": "int", "default": "2048", "required": false, "description": "Chunk size for prompt prefill processing (default: 2048)" }, { "name": "specprefill_enabled", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable SpecPrefill (attention-based sparse prefill)" }, { "name": "specprefill_threshold", "kind": "positional or keyword", "annotation": "int", "default": "8192", "required": false, "description": "Minimum suffix tokens to trigger SpecPrefill" }, { "name": "specprefill_keep_pct", "kind": "positional or keyword", "annotation": "float", "default": "0.3", "required": false, "description": "Fraction of tokens to keep (default: 0.3)" }, { "name": "specprefill_backbone_pct", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Fraction of chunks to reserve for evenly spaced coverage (default: 0.0)" }, { "name": "specprefill_draft_model", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Path to small draft model for importance scoring" }, { "name": "max_kv_size", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Maximum KV cache size per sequence (0 = unbounded)" }, { "name": "mllm_draft_model", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional MLLM speculative draft/assistant model path" }, { "name": "mllm_draft_kind", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional mlx-vlm draft kind, for example \"mtp\"" }, { "name": "mllm_draft_block_size", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional speculative block size for mlx-vlm" } ], "return_annotation": "SimpleEngine", "docstring": "Simple engine for direct model calls.\n\nThis engine provides maximum throughput for single-user scenarios\nby calling mlx-lm/mlx-vlm directly without batching overhead.", "summary": "Simple engine for direct model calls.", "implementation": "Class `SimpleEngine` derives from `BaseEngine` and declares 31 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 122, "end_line": 2912, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L122-L2912", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "SimpleEngine.__init__", "full_name": "vllm_mlx.engine.simple.SimpleEngine.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, trust_remote_code: bool=False, enable_cache: bool=True, force_mllm: bool=False, mtp: bool=False, mtp_num_draft_tokens: int=1, prefill_step_size: int=2048, specprefill_enabled: bool=False, specprefill_threshold: int=8192, specprefill_keep_pct: float=0.3, specprefill_backbone_pct: float=0.0, specprefill_draft_model: str | None=None, max_kv_size: int=0, mllm_draft_model: str | None=None, mllm_draft_kind: str | None=None, mllm_draft_block_size: int | None=None)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "enable_cache", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Enable VLM cache for multimodal models" }, { "name": "force_mllm", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Force loading as MLLM even if not auto-detected" }, { "name": "mtp", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable native MTP speculative decoding (model must have MTP head)" }, { "name": "mtp_num_draft_tokens", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Draft tokens per speculative MTP step" }, { "name": "prefill_step_size", "kind": "positional or keyword", "annotation": "int", "default": "2048", "required": false, "description": "Chunk size for prompt prefill processing (default: 2048)" }, { "name": "specprefill_enabled", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable SpecPrefill (attention-based sparse prefill)" }, { "name": "specprefill_threshold", "kind": "positional or keyword", "annotation": "int", "default": "8192", "required": false, "description": "Minimum suffix tokens to trigger SpecPrefill" }, { "name": "specprefill_keep_pct", "kind": "positional or keyword", "annotation": "float", "default": "0.3", "required": false, "description": "Fraction of tokens to keep (default: 0.3)" }, { "name": "specprefill_backbone_pct", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Fraction of chunks to reserve for evenly spaced coverage (default: 0.0)" }, { "name": "specprefill_draft_model", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Path to small draft model for importance scoring" }, { "name": "max_kv_size", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Maximum KV cache size per sequence (0 = unbounded)" }, { "name": "mllm_draft_model", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional MLLM speculative draft/assistant model path" }, { "name": "mllm_draft_kind", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional mlx-vlm draft kind, for example \"mtp\"" }, { "name": "mllm_draft_block_size", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional speculative block size for mlx-vlm" } ], "return_annotation": "not annotated", "docstring": "Initialize the simple engine.\n\nArgs:\n model_name: HuggingFace model name or local path\n trust_remote_code: Whether to trust remote code\n enable_cache: Enable VLM cache for multimodal models\n force_mllm: Force loading as MLLM even if not auto-detected\n mtp: Enable native MTP speculative decoding (model must have MTP head)\n mtp_num_draft_tokens: Draft tokens per speculative MTP step\n prefill_step_size: Chunk size for prompt prefill processing (default: 2048)\n specprefill_enabled: Enable SpecPrefill (attention-based sparse prefill)\n specprefill_threshold: Minimum suffix tokens to trigger SpecPrefill\n specprefill_keep_pct: Fraction of tokens to keep (default: 0.3)\n specprefill_backbone_pct: Fraction of chunks to reserve for evenly\n spaced coverage (default: 0.0)\n specprefill_draft_model: Path to small draft model for importance scoring\n max_kv_size: Maximum KV cache size per sequence (0 = unbounded)\n mllm_draft_model: Optional MLLM speculative draft/assistant model path\n mllm_draft_kind: Optional mlx-vlm draft kind, for example \"mtp\"\n mllm_draft_block_size: Optional speculative block size for mlx-vlm", "summary": "Initialize the simple engine.", "implementation": "Method `SimpleEngine.__init__` updates `self._model_name`, `self._created_at`, `self._trust_remote_code`, `self._enable_cache`; calls `time.time`, `is_mllm_model`, `deque`, `asyncio.Lock`.", "documented": true, "public": false, "addressable": true, "line": 130, "end_line": 257, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L130-L257", "decorators": [], "calls": [ "time.time", "is_mllm_model", "deque", "asyncio.Lock", "os.environ.get('VLLM_MLX_SIMPLE_ENGINE_LOCK_ADMISSION', 'fail_fast').strip().lower", "os.environ.get('VLLM_MLX_SIMPLE_ENGINE_LOCK_ADMISSION', 'fail_fast').strip", "os.environ.get", "logger.warning", "max", "int", "OrderedDict" ], "state_reads": [ "self._generation_lock_admission" ], "state_writes": [ "self._model_name", "self._created_at", "self._trust_remote_code", "self._enable_cache", "self._is_mllm", "self._mtp", "self._mtp_num_draft_tokens", "self._prefill_step_size", "self._total_requests_processed", "self._total_prompt_tokens", "self._total_completion_tokens", "self._num_running", "self._recent_completions", "self._active_requests", "self._specprefill_enabled", "self._specprefill_threshold", "self._specprefill_keep_pct", "self._specprefill_backbone_pct", "self._specprefill_draft_model_path", "self._mllm_draft_model_path", "self._mllm_draft_kind", "self._mllm_draft_block_size", "self._max_kv_size", "self._model", "self._loaded", "self._text_model", "self._text_tokenizer", "self._draft_model", "self._generation_lock", "self._generation_lock_admission", "self._generation_waiters", "self._generation_busy_rejections", "self._system_kv_capacity", "self._system_kv_cache", "self._system_kv_cache_stats", "self._supports_system_kv_cache" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_clone_cache_state", "qualname": "SimpleEngine._clone_cache_state", "full_name": "vllm_mlx.engine.simple.SimpleEngine._clone_cache_state", "kind": "method", "signature": "def _clone_cache_state(value: Any) -> Any", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Copy cache state containers without duplicating immutable MLX arrays.", "summary": "Copy cache state containers without duplicating immutable MLX arrays.", "implementation": "Method `SimpleEngine._clone_cache_state` calls `isinstance`, `tuple`, `SimpleEngine._clone_cache_state`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 260, "end_line": 266, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L260-L266", "decorators": [ "staticmethod" ], "calls": [ "isinstance", "tuple", "SimpleEngine._clone_cache_state" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tuple((SimpleEngine._clone_cache_state(v) for v in value))", "[SimpleEngine._clone_cache_state(v) for v in value]", "value" ], "awaits": false, "yields": false }, { "name": "_snapshot_prompt_cache", "qualname": "SimpleEngine._snapshot_prompt_cache", "full_name": "vllm_mlx.engine.simple.SimpleEngine._snapshot_prompt_cache", "kind": "method", "signature": "def _snapshot_prompt_cache(cls, prompt_cache: list[Any]) -> list[Any]", "parameters": [ { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[Any]", "docstring": "Capture cache states without aliasing mutable state containers.", "summary": "Capture cache states without aliasing mutable state containers.", "implementation": "Method `SimpleEngine._snapshot_prompt_cache` calls `cls._clone_cache_state`; returns `[cls._clone_cache_state(c.state) for c in prompt_cache]`.", "documented": true, "public": false, "addressable": true, "line": 269, "end_line": 271, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L269-L271", "decorators": [ "classmethod" ], "calls": [ "cls._clone_cache_state" ], "state_reads": [ "cls._clone_cache_state" ], "state_writes": [], "raises": [], "return_expressions": [ "[cls._clone_cache_state(c.state) for c in prompt_cache]" ], "awaits": false, "yields": false }, { "name": "_restore_prompt_cache", "qualname": "SimpleEngine._restore_prompt_cache", "full_name": "vllm_mlx.engine.simple.SimpleEngine._restore_prompt_cache", "kind": "method", "signature": "def _restore_prompt_cache(cls, prompt_cache: list[Any], snapshot: list[Any]) -> None", "parameters": [ { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "snapshot", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Restore cache states without letting decode mutate the saved snapshot.", "summary": "Restore cache states without letting decode mutate the saved snapshot.", "implementation": "Method `SimpleEngine._restore_prompt_cache` calls `enumerate`, `cls._clone_cache_state`.", "documented": true, "public": false, "addressable": true, "line": 274, "end_line": 279, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L274-L279", "decorators": [ "classmethod" ], "calls": [ "enumerate", "cls._clone_cache_state" ], "state_reads": [ "cls._clone_cache_state" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_iter_cache_state_arrays", "qualname": "SimpleEngine._iter_cache_state_arrays", "full_name": "vllm_mlx.engine.simple.SimpleEngine._iter_cache_state_arrays", "kind": "method", "signature": "def _iter_cache_state_arrays(value: Any)", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `SimpleEngine._iter_cache_state_arrays` calls `isinstance`, `SimpleEngine._iter_cache_state_arrays`, `hasattr`; yields values incrementally.", "implementation": "Method `SimpleEngine._iter_cache_state_arrays` calls `isinstance`, `SimpleEngine._iter_cache_state_arrays`, `hasattr`; yields values incrementally.", "documented": false, "public": false, "addressable": true, "line": 282, "end_line": 287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L282-L287", "decorators": [ "staticmethod" ], "calls": [ "isinstance", "SimpleEngine._iter_cache_state_arrays", "hasattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "_eval_cache_snapshot", "qualname": "SimpleEngine._eval_cache_snapshot", "full_name": "vllm_mlx.engine.simple.SimpleEngine._eval_cache_snapshot", "kind": "method", "signature": "def _eval_cache_snapshot(cls, snapshot: list[Any]) -> None", "parameters": [ { "name": "snapshot", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `SimpleEngine._eval_cache_snapshot` calls `list`, `cls._iter_cache_state_arrays`, `mx.eval`.", "implementation": "Method `SimpleEngine._eval_cache_snapshot` calls `list`, `cls._iter_cache_state_arrays`, `mx.eval`.", "documented": false, "public": false, "addressable": true, "line": 290, "end_line": 293, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L290-L293", "decorators": [ "classmethod" ], "calls": [ "list", "cls._iter_cache_state_arrays", "mx.eval" ], "state_reads": [ "cls._iter_cache_state_arrays" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_cache_class_is_system_snapshot_safe", "qualname": "SimpleEngine._cache_class_is_system_snapshot_safe", "full_name": "vllm_mlx.engine.simple.SimpleEngine._cache_class_is_system_snapshot_safe", "kind": "method", "signature": "def _cache_class_is_system_snapshot_safe(cache_entry: Any) -> bool", "parameters": [ { "name": "cache_entry", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "", "summary": "Method `SimpleEngine._cache_class_is_system_snapshot_safe` calls `isinstance`, `type`; has 2 explicit return paths.", "implementation": "Method `SimpleEngine._cache_class_is_system_snapshot_safe` calls `isinstance`, `type`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 296, "end_line": 303, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L296-L303", "decorators": [ "staticmethod" ], "calls": [ "isinstance", "type" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "isinstance(cache_entry, (KVCache, ArraysCache))", "cache_type in {'KVCache', 'ArraysCache'}" ], "awaits": false, "yields": false }, { "name": "_probe_system_kv_cache_support", "qualname": "SimpleEngine._probe_system_kv_cache_support", "full_name": "vllm_mlx.engine.simple.SimpleEngine._probe_system_kv_cache_support", "kind": "method", "signature": "def _probe_system_kv_cache_support(cls, model: Any, route: str) -> bool", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "route", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "", "summary": "Method `SimpleEngine._probe_system_kv_cache_support` calls `make_prompt_cache`, `bool`, `all`, `cls._cache_class_is_system_snapshot_safe`; has 2 explicit return paths.", "implementation": "Method `SimpleEngine._probe_system_kv_cache_support` calls `make_prompt_cache`, `bool`, `all`, `cls._cache_class_is_system_snapshot_safe`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 306, "end_line": 331, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L306-L331", "decorators": [ "classmethod" ], "calls": [ "make_prompt_cache", "bool", "all", "cls._cache_class_is_system_snapshot_safe", "sorted", "type", "logger.info", "logger.debug" ], "state_reads": [ "cls._cache_class_is_system_snapshot_safe" ], "state_writes": [], "raises": [], "return_expressions": [ "supported", "False" ], "awaits": false, "yields": false }, { "name": "model_name", "qualname": "SimpleEngine.model_name", "full_name": "vllm_mlx.engine.simple.SimpleEngine.model_name", "kind": "method", "signature": "def model_name(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Get the model name.", "summary": "Get the model name.", "implementation": "Method `SimpleEngine.model_name` returns `self._model_name`.", "documented": true, "public": true, "addressable": true, "line": 334, "end_line": 336, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L334-L336", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._model_name" ], "state_writes": [], "raises": [], "return_expressions": [ "self._model_name" ], "awaits": false, "yields": false }, { "name": "is_mllm", "qualname": "SimpleEngine.is_mllm", "full_name": "vllm_mlx.engine.simple.SimpleEngine.is_mllm", "kind": "method", "signature": "def is_mllm(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if this is a multimodal model.", "summary": "Check if this is a multimodal model.", "implementation": "Method `SimpleEngine.is_mllm` returns `self._is_mllm`.", "documented": true, "public": true, "addressable": true, "line": 339, "end_line": 341, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L339-L341", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._is_mllm" ], "state_writes": [], "raises": [], "return_expressions": [ "self._is_mllm" ], "awaits": false, "yields": false }, { "name": "tokenizer", "qualname": "SimpleEngine.tokenizer", "full_name": "vllm_mlx.engine.simple.SimpleEngine.tokenizer", "kind": "method", "signature": "def tokenizer(self) -> Any", "parameters": [], "return_annotation": "Any", "docstring": "Get the tokenizer.", "summary": "Get the tokenizer.", "implementation": "Method `SimpleEngine.tokenizer` calls `getattr`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 344, "end_line": 350, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L344-L350", "decorators": [ "property" ], "calls": [ "getattr" ], "state_reads": [ "self._loaded", "self._model", "self._is_mllm", "self._model.tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "getattr(self._model, 'processor', None)", "self._model.tokenizer" ], "awaits": false, "yields": false }, { "name": "_generation_lock_holder_summary", "qualname": "SimpleEngine._generation_lock_holder_summary", "full_name": "vllm_mlx.engine.simple.SimpleEngine._generation_lock_holder_summary", "kind": "method", "signature": "def _generation_lock_holder_summary(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `SimpleEngine._generation_lock_holder_summary` calls `time.time`, `self._active_requests.items`, `info.get`, `round`; has 2 explicit return paths.", "implementation": "Method `SimpleEngine._generation_lock_holder_summary` calls `time.time`, `self._active_requests.items`, `info.get`, `round`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 352, "end_line": 371, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L352-L371", "decorators": [], "calls": [ "time.time", "self._active_requests.items", "info.get", "round", "holders.append", "','.join" ], "state_reads": [ "self._active_requests", "self._active_requests.items" ], "state_writes": [], "raises": [], "return_expressions": [ "'none'", "','.join(holders)" ], "awaits": false, "yields": false }, { "name": "_acquire_generation_slot", "qualname": "SimpleEngine._acquire_generation_slot", "full_name": "vllm_mlx.engine.simple.SimpleEngine._acquire_generation_slot", "kind": "method", "signature": "async def _acquire_generation_slot(self, request_id: str)", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Admission control for SimpleEngine's serialized MLX route.", "summary": "Admission control for SimpleEngine's serialized MLX route.", "implementation": "Method `SimpleEngine._acquire_generation_slot` updates `self._generation_busy_rejections`, `self._generation_waiters`; calls `self._generation_lock.locked`, `EngineBusy`, `self._generation_lock_holder_summary`; yields values incrementally; can raise `EngineBusy`.", "documented": true, "public": false, "addressable": true, "line": 374, "end_line": 398, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L374-L398", "decorators": [ "asynccontextmanager" ], "calls": [ "self._generation_lock.locked", "EngineBusy", "self._generation_lock_holder_summary" ], "state_reads": [ "self._generation_lock_admission", "self._generation_lock.locked", "self._generation_lock", "self._generation_lock_holder_summary", "self._generation_waiters" ], "state_writes": [ "self._generation_busy_rejections", "self._generation_waiters" ], "raises": [ "EngineBusy" ], "return_expressions": [], "awaits": false, "yields": true }, { "name": "prepare_for_start", "qualname": "SimpleEngine.prepare_for_start", "full_name": "vllm_mlx.engine.simple.SimpleEngine.prepare_for_start", "kind": "method", "signature": "def prepare_for_start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the backing model off the serving event loop.", "summary": "Load the backing model off the serving event loop.", "implementation": "Method `SimpleEngine.prepare_for_start` updates `self._model`; calls `MLXMultimodalLM`, `MLXLanguageModel`, `self._model.load`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 400, "end_line": 427, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L400-L427", "decorators": [], "calls": [ "MLXMultimodalLM", "MLXLanguageModel", "self._model.load" ], "state_reads": [ "self._model", "self._is_mllm", "self._model_name", "self._trust_remote_code", "self._enable_cache", "self._max_kv_size", "self._mllm_draft_model_path", "self._mllm_draft_kind", "self._mllm_draft_block_size", "self._mtp", "self._mtp_num_draft_tokens", "self._model.load" ], "state_writes": [ "self._model" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_uses_default_prepare_for_start", "qualname": "SimpleEngine._uses_default_prepare_for_start", "full_name": "vllm_mlx.engine.simple.SimpleEngine._uses_default_prepare_for_start", "kind": "method", "signature": "def _uses_default_prepare_for_start(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return True when prepare_for_start is the class implementation.", "summary": "Return True when prepare_for_start is the class implementation.", "implementation": "Method `SimpleEngine._uses_default_prepare_for_start` calls `getattr`; returns `method is SimpleEngine.prepare_for_start`.", "documented": true, "public": false, "addressable": true, "line": 429, "end_line": 432, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L429-L432", "decorators": [], "calls": [ "getattr" ], "state_reads": [ "self.prepare_for_start" ], "state_writes": [], "raises": [], "return_expressions": [ "method is SimpleEngine.prepare_for_start" ], "awaits": false, "yields": false }, { "name": "start", "qualname": "SimpleEngine.start", "full_name": "vllm_mlx.engine.simple.SimpleEngine.start", "kind": "method", "signature": "async def start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the engine (load model if not loaded).", "summary": "Start the engine (load model if not loaded).", "implementation": "Method `SimpleEngine.start` updates `self._loaded`, `self._supports_system_kv_cache`, `self._text_model`, `self._text_tokenizer`; calls `self._uses_default_prepare_for_start`, `self.prepare_for_start`, `run_blocking_startup_work`, `logger.warning`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 434, "end_line": 595, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L434-L595", "decorators": [], "calls": [ "self._uses_default_prepare_for_start", "self.prepare_for_start", "run_blocking_startup_work", "logger.warning", "getattr", "self._probe_system_kv_cache_support", "self._should_route_text_through_text_model", "build_text_model", "self._model.get_tokenizer", "self._model_name.lower", "self._text_tokenizer.convert_tokens_to_ids", "make_prompt_cache", "bool", "all", "isinstance", "sorted", "type", "logger.info", "logger.debug", "hasattr", "logger.error", "mlx_lm_load", "cleanup_startup_cancellation" ], "state_reads": [ "self._loaded", "self._model", "self._uses_default_prepare_for_start", "self.prepare_for_start", "self._mtp", "self._mtp_num_draft_tokens", "self._is_mllm", "self._probe_system_kv_cache_support", "self._should_route_text_through_text_model", "self._model.model", "self._model_name", "self._text_model", "self._model.get_tokenizer", "self._model_name.lower", "self._text_tokenizer", "self._text_tokenizer.convert_tokens_to_ids", "self._max_kv_size", "self._supports_system_kv_cache", "self._text_model.mtp", "self._specprefill_enabled", "self._specprefill_draft_model_path", "self._specprefill_threshold", "self._specprefill_keep_pct", "self._draft_model", "self.stop" ], "state_writes": [ "self._loaded", "self._supports_system_kv_cache", "self._text_model", "self._text_tokenizer", "self._text_tokenizer.eos_token", "self._text_tokenizer.eos_token_id", "self._draft_model" ], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "stop", "qualname": "SimpleEngine.stop", "full_name": "vllm_mlx.engine.simple.SimpleEngine.stop", "kind": "method", "signature": "async def stop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop the engine and cleanup resources.", "summary": "Stop the engine and cleanup resources.", "implementation": "Method `SimpleEngine.stop` updates `self._model`, `self._text_model`, `self._text_tokenizer`, `self._draft_model`; calls `self._system_kv_cache.clear`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 597, "end_line": 608, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L597-L608", "decorators": [], "calls": [ "self._system_kv_cache.clear", "logger.info" ], "state_reads": [ "self._system_kv_cache.clear", "self._system_kv_cache", "self._system_kv_cache_stats" ], "state_writes": [ "self._model", "self._text_model", "self._text_tokenizer", "self._draft_model", "self._loaded", "self._supports_system_kv_cache" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_should_route_text_through_text_model", "qualname": "SimpleEngine._should_route_text_through_text_model", "full_name": "vllm_mlx.engine.simple.SimpleEngine._should_route_text_through_text_model", "kind": "method", "signature": "def _should_route_text_through_text_model(self, *, mllm_draft_requested: bool=False) -> bool", "parameters": [ { "name": "mllm_draft_requested", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." } ], "return_annotation": "bool", "docstring": "Return whether text-only MLLM requests may use mlx_lm TextModel.", "summary": "Return whether text-only MLLM requests may use mlx_lm TextModel.", "implementation": "Method `SimpleEngine._should_route_text_through_text_model` returns `not (mllm_draft_requested and self._mllm_draft_model_path is not None)`.", "documented": true, "public": false, "addressable": true, "line": 610, "end_line": 614, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L610-L614", "decorators": [], "calls": [], "state_reads": [ "self._mllm_draft_model_path" ], "state_writes": [], "raises": [], "return_expressions": [ "not (mllm_draft_requested and self._mllm_draft_model_path is not None)" ], "awaits": false, "yields": false }, { "name": "_run_blocking_serialized", "qualname": "SimpleEngine._run_blocking_serialized", "full_name": "vllm_mlx.engine.simple.SimpleEngine._run_blocking_serialized", "kind": "method", "signature": "async def _run_blocking_serialized(self, func, /, *args, request_id: str | None=None, on_cancel=None, **kwargs)", "parameters": [ { "name": "func", "kind": "positional-only", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional-only input." }, { "name": "*args", "kind": "variadic positional", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic positional inputs accepted by this callable." }, { "name": "request_id", "kind": "keyword-only", "annotation": "str | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "on_cancel", "kind": "keyword-only", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "not annotated", "docstring": "Run a blocking MLX operation under the generation lock.\n\nCancellation must not release the async lock before the worker thread\nfinishes, or a follow-up request can enter MLX/Metal concurrently and\ncorrupt the command-buffer state.", "summary": "Run a blocking MLX operation under the generation lock.", "implementation": "Method `SimpleEngine._run_blocking_serialized` calls `id`, `self._acquire_generation_slot`, `time.time`, `asyncio.create_task`; awaits asynchronous work; returns `await asyncio.shield(task)`.", "documented": true, "public": false, "addressable": true, "line": 616, "end_line": 666, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L616-L666", "decorators": [], "calls": [ "id", "self._acquire_generation_slot", "time.time", "asyncio.create_task", "asyncio.to_thread", "asyncio.shield", "on_cancel", "logger.debug", "self._active_requests.pop" ], "state_reads": [ "self._acquire_generation_slot", "self._active_requests", "self._active_requests.pop" ], "state_writes": [], "raises": [], "return_expressions": [ "await asyncio.shield(task)" ], "awaits": true, "yields": false }, { "name": "run_bound", "qualname": "SimpleEngine._run_blocking_serialized.run_bound", "full_name": "vllm_mlx.engine.simple.SimpleEngine._run_blocking_serialized.run_bound", "kind": "nested function", "signature": "def run_bound()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `SimpleEngine._run_blocking_serialized.run_bound` calls `_bind_worker_generation_streams`, `func`; returns `func(*args, **kwargs)`.", "implementation": "Nested Function `SimpleEngine._run_blocking_serialized.run_bound` calls `_bind_worker_generation_streams`, `func`; returns `func(*args, **kwargs)`.", "documented": false, "public": false, "addressable": false, "line": 644, "end_line": 646, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L644-L646", "decorators": [], "calls": [ "_bind_worker_generation_streams", "func" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "func(*args, **kwargs)" ], "awaits": false, "yields": false }, { "name": "generate", "qualname": "SimpleEngine.generate", "full_name": "vllm_mlx.engine.simple.SimpleEngine.generate", "kind": "method", "signature": "async def generate(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, stop: list[str] | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Input text" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Stop sequences" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters forwarded to stream_generate, including per-request `specprefill` / `specprefill_keep_pct`" } ], "return_annotation": "GenerationOutput", "docstring": "Generate a complete response (non-streaming).\n\nThin accumulator over stream_generate(). stream_generate() is the\nonly code path that consumes per-request SpecPrefill overrides\n(`specprefill`, `specprefill_keep_pct`) and routes through\n_stream_generate_specprefill() when engaged. The prior direct\nself._model.generate() path silently dropped those overrides for\nnon-streaming /v1/completions callers, so extra_body.specprefill\nwas advertised by the server but had no effect on this route.\n\nBy iterating stream_generate() and returning the last\nGenerationOutput, non-streaming clients get the same SpecPrefill\nengagement, accurate prompt_tokens reporting, and per-request\noverride support as streaming clients.\n\nArgs:\n prompt: Input text\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n stop: Stop sequences\n **kwargs: Additional parameters forwarded to stream_generate,\n including per-request `specprefill` / `specprefill_keep_pct`\n\nReturns:\n GenerationOutput with complete text", "summary": "Generate a complete response (non-streaming).", "implementation": "Method `SimpleEngine.generate` calls `self.start`, `self.stream_generate`, `GenerationOutput`, `clean_output_text`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 668, "end_line": 730, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L668-L730", "decorators": [], "calls": [ "self.start", "self.stream_generate", "GenerationOutput", "clean_output_text", "list" ], "state_reads": [ "self._loaded", "self.start", "self.stream_generate" ], "state_writes": [], "raises": [], "return_expressions": [ "GenerationOutput(text='', finish_reason='stop')", "GenerationOutput(text=text, tokens=list(last_output.tokens), prompt_tokens=last_output.prompt_tokens, completion_tokens…" ], "awaits": true, "yields": false }, { "name": "_track_request_stream", "qualname": "SimpleEngine._track_request_stream", "full_name": "vllm_mlx.engine.simple.SimpleEngine._track_request_stream", "kind": "method", "signature": "async def _track_request_stream(self, source_gen: AsyncIterator[GenerationOutput], *, max_tokens: int=0) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "source_gen", "kind": "positional or keyword", "annotation": "AsyncIterator[GenerationOutput]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "keyword-only", "annotation": "int", "default": "0", "required": false, "description": "Optional keyword-only input; defaults to `0`." } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Yield-through wrapper that records per-request live state and\nfinal ``prompt_tokens``/``completion_tokens`` counters.\n\nMirrors the fields BatchedEngine emits per running request\n(``request_id``, ``phase``, ``elapsed_s``, ``ttft_s``,\n``tokens_per_second``, ``progress``, ...) so dashboards built\nagainst ``/v1/status`` show individual in-flight requests for\nSimpleEngine-backed services as well (Gemma 4 31B + MTP, etc.).\n\nRe-entrant calls (e.g. the cache-fallback path inside\n``_stream_chat_impl`` that delegates to ``self.stream_generate``)\nare detected via the ``_in_tracker`` context variable and pass\nthrough without a second tracking entry, so each external\nrequest is counted exactly once.\n\nNote: we deliberately use ``set(True)``/``set(False)`` rather\nthan ``set(token)``/``reset(token)``. FastAPI/uvicorn finalize\nstreaming generators from a different async context than the\none that created them; ``ContextVar.reset(token)`` raises\n``ValueError`` in that case (\"Token was created in a different\nContext\"), which surfaces as a terminal-frame streaming error.\n``set(False)`` works in any context and the contextvar is only\nconsumed inside this method, so there is no value to preserve.", "summary": "Yield-through wrapper that records per-request live state and final ``prompt_tokens``/``completion_tokens`` counters.", "implementation": "Method `SimpleEngine._track_request_stream` updates `self._num_running`, `self._total_requests_processed`, `self._total_prompt_tokens`, `self._total_completion_tokens`; calls `_in_tracker.get`, `_in_tracker.set`, `str`, `uuid.uuid4`; yields values incrementally; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 732, "end_line": 817, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L732-L817", "decorators": [], "calls": [ "_in_tracker.get", "_in_tracker.set", "str", "uuid.uuid4", "time.time", "hasattr", "round", "min", "max", "self._active_requests.pop", "self._recent_completions.append" ], "state_reads": [ "self._active_requests", "self._active_requests.pop", "self._num_running", "self._recent_completions.append", "self._recent_completions" ], "state_writes": [ "self._num_running", "self._total_requests_processed", "self._total_prompt_tokens", "self._total_completion_tokens" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": true }, { "name": "stream_generate", "qualname": "SimpleEngine.stream_generate", "full_name": "vllm_mlx.engine.simple.SimpleEngine.stream_generate", "kind": "method", "signature": "async def stream_generate(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, stop: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Optional positional or keyword input; defaults to `256`." }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Optional positional or keyword input; defaults to `0.7`." }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Optional positional or keyword input; defaults to `0.9`." }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Public stream-generate wrapper with request stats tracking.", "summary": "Public stream-generate wrapper with request stats tracking.", "implementation": "Method `SimpleEngine.stream_generate` calls `self._track_request_stream`, `self._stream_generate_impl`; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 819, "end_line": 840, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L819-L840", "decorators": [], "calls": [ "self._track_request_stream", "self._stream_generate_impl" ], "state_reads": [ "self._track_request_stream", "self._stream_generate_impl" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "_stream_generate_impl", "qualname": "SimpleEngine._stream_generate_impl", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_impl", "kind": "method", "signature": "async def _stream_generate_impl(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, stop: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Input text" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Stop sequences" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Stream generation token by token.\n\nArgs:\n prompt: Input text\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n stop: Stop sequences\n **kwargs: Additional model-specific parameters\n\nYields:\n GenerationOutput with incremental text", "summary": "Stream generation token by token.", "implementation": "Method `SimpleEngine._stream_generate_impl` calls `self.start`, `kwargs.pop`, `str`, `id`; awaits asynchronous work; yields values incrementally; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 842, "end_line": 1012, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L842-L1012", "decorators": [], "calls": [ "self.start", "kwargs.pop", "str", "id", "prompt.startswith", "tokenizer.encode", "len", "logger.warning", "self._stream_generate_specprefill", "self._acquire_generation_slot", "time.time", "_bind_worker_generation_streams", "self._model.stream_generate", "hasattr", "self._active_requests[request_id].update", "round", "getattr", "GenerationOutput", "self._model.tokenizer.encode", "self._active_requests.pop" ], "state_reads": [ "self._loaded", "self.start", "self._is_mllm", "self._draft_model", "self._model.tokenizer", "self._model", "self._specprefill_threshold", "self._stream_generate_specprefill", "self._acquire_generation_slot", "self._active_requests", "self._model.stream_generate", "self._model.tokenizer.encode", "self._active_requests.pop" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": true }, { "name": "chat", "qualname": "SimpleEngine.chat", "full_name": "vllm_mlx.engine.simple.SimpleEngine.chat", "kind": "method", "signature": "async def chat(self, messages: list[dict[str, Any]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list[dict] | None=None, images: list[str] | None=None, videos: list[str] | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "List of chat messages" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional tool definitions" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "GenerationOutput", "docstring": "Chat completion (non-streaming).\n\nArgs:\n messages: List of chat messages\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n tools: Optional tool definitions\n images: Optional image URLs/paths\n videos: Optional video URLs/paths\n **kwargs: Additional model-specific parameters\n\nReturns:\n GenerationOutput with assistant response", "summary": "Chat completion (non-streaming).", "implementation": "Method `SimpleEngine.chat` calls `self.start`, `dict`, `kwargs.pop`, `aggregate_stream_chat`; awaits asynchronous work; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1014, "end_line": 1144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1014-L1144", "decorators": [], "calls": [ "self.start", "dict", "kwargs.pop", "aggregate_stream_chat", "kwargs.get", "has_media_content", "convert_tools_for_template", "self._run_blocking_serialized", "clean_output_text", "GenerationOutput", "getattr", "tokenizer.apply_chat_template", "len" ], "state_reads": [ "self._loaded", "self.start", "self._is_mllm", "self._run_blocking_serialized", "self._model.chat", "self._model", "self._model.tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "await aggregate_stream_chat()", "GenerationOutput(text=text, prompt_tokens=output.prompt_tokens, completion_tokens=output.completion_tokens, finish_reas…", "GenerationOutput(text=text, tokens=output.tokens, prompt_tokens=prompt_token_count, completion_tokens=len(output.tokens…" ], "awaits": true, "yields": false }, { "name": "aggregate_stream_chat", "qualname": "SimpleEngine.chat.aggregate_stream_chat", "full_name": "vllm_mlx.engine.simple.SimpleEngine.chat.aggregate_stream_chat", "kind": "nested function", "signature": "async def aggregate_stream_chat() -> GenerationOutput", "parameters": [], "return_annotation": "GenerationOutput", "docstring": "", "summary": "Nested Function `SimpleEngine.chat.aggregate_stream_chat` calls `GenerationOutput`, `self.stream_chat`, `clean_output_text`, `list`; returns `GenerationOutput(text=text, tokens=list(final_output.tokens), prompt_tokens=final_output.prompt_tokens, completion_toke…`.", "implementation": "Nested Function `SimpleEngine.chat.aggregate_stream_chat` calls `GenerationOutput`, `self.stream_chat`, `clean_output_text`, `list`; returns `GenerationOutput(text=text, tokens=list(final_output.tokens), prompt_tokens=final_output.prompt_tokens, completion_toke…`.", "documented": false, "public": false, "addressable": false, "line": 1046, "end_line": 1069, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1046-L1069", "decorators": [], "calls": [ "GenerationOutput", "self.stream_chat", "clean_output_text", "list" ], "state_reads": [ "self.stream_chat" ], "state_writes": [], "raises": [], "return_expressions": [ "GenerationOutput(text=text, tokens=list(final_output.tokens), prompt_tokens=final_output.prompt_tokens, completion_toke…" ], "awaits": false, "yields": false }, { "name": "stream_chat", "qualname": "SimpleEngine.stream_chat", "full_name": "vllm_mlx.engine.simple.SimpleEngine.stream_chat", "kind": "method", "signature": "async def stream_chat(self, messages: list[dict[str, Any]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list[dict] | None=None, images: list[str] | None=None, videos: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Optional positional or keyword input; defaults to `256`." }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Optional positional or keyword input; defaults to `0.7`." }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Optional positional or keyword input; defaults to `0.9`." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Public stream-chat wrapper with request stats tracking.", "summary": "Public stream-chat wrapper with request stats tracking.", "implementation": "Method `SimpleEngine.stream_chat` calls `self._track_request_stream`, `self._stream_chat_impl`; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 1146, "end_line": 1171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1146-L1171", "decorators": [], "calls": [ "self._track_request_stream", "self._stream_chat_impl" ], "state_reads": [ "self._track_request_stream", "self._stream_chat_impl" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "_stream_chat_impl", "qualname": "SimpleEngine._stream_chat_impl", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl", "kind": "method", "signature": "async def _stream_chat_impl(self, messages: list[dict[str, Any]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list[dict] | None=None, images: list[str] | None=None, videos: list[str] | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "List of chat messages" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional tool definitions" }, { "name": "images", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional image URLs/paths" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional video URLs/paths" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Stream chat completion token by token.\n\nArgs:\n messages: List of chat messages\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n tools: Optional tool definitions\n images: Optional image URLs/paths\n videos: Optional video URLs/paths\n **kwargs: Additional model-specific parameters\n\nYields:\n GenerationOutput with incremental text", "summary": "Stream chat completion token by token.", "implementation": "Method `SimpleEngine._stream_chat_impl` calls `self.start`, `dict`, `kwargs.pop`, `bool`; awaits asynchronous work; yields values incrementally; can raise `payload`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1173, "end_line": 1794, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1173-L1794", "decorators": [], "calls": [ "self.start", "dict", "kwargs.pop", "bool", "has_media_content", "convert_tools_for_template", "self._should_route_text_through_text_model", "hasattr", "logger.info", "self._stream_generate_text", "str", "id", "getattr", "self._model._collect_video_inputs", "mllm_call_kwargs", "self._acquire_generation_slot", "_bind_worker_generation_streams", "self._model.stream_chat", "GenerationOutput", "self._run_blocking_serialized", "self._model_name.lower", "template_kwargs.update", "normalize_messages_for_chat_template", "chat_template_kwargs.get", "_harmony_render_messages", "tokenizer.apply_chat_template", "chat_template_kwargs.keys", "'\\n'.join", "kwargs.get", "cache_blocking_controls.append", "_to_msg_dict", "any", "m.get", "_with_user", "isinstance", "range", "min", "len", "hashlib.sha256(system_prefix_text.encode()).hexdigest", "hashlib.sha256", "system_prefix_text.encode", "prompt.startswith", "tokenizer.encode", "self._system_kv_cache.get", "asyncio.get_running_loop", "asyncio.Queue", "threading.Event", "asyncio.create_task", "_produce_responses", "response_queue.get", "logger.warning", "producer_task.done", "abort_event.set", "self.stream_generate" ], "state_reads": [ "self._loaded", "self.start", "self._is_mllm", "self._text_model", "self._should_route_text_through_text_model", "self._text_model.mtp", "self._mtp", "self._stream_generate_text", "self._model", "self._model._collect_video_inputs", "self._acquire_generation_slot", "self._model.stream_chat", "self._run_blocking_serialized", "self._model.tokenizer", "self._model_name.lower", "self._model_name", "self._draft_model", "self._max_kv_size", "self._supports_system_kv_cache", "self._system_kv_cache.get", "self._system_kv_cache", "self.stream_generate" ], "state_writes": [], "raises": [ "payload" ], "return_expressions": [ "None" ], "awaits": true, "yields": true }, { "name": "mllm_call_kwargs", "qualname": "SimpleEngine._stream_chat_impl.mllm_call_kwargs", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl.mllm_call_kwargs", "kind": "nested function", "signature": "def mllm_call_kwargs() -> dict", "parameters": [], "return_annotation": "dict", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl.mllm_call_kwargs` calls `dict`; returns `local_kwargs`.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl.mllm_call_kwargs` calls `dict`; returns `local_kwargs`.", "documented": false, "public": false, "addressable": false, "line": 1236, "end_line": 1242, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1236-L1242", "decorators": [], "calls": [ "dict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "local_kwargs" ], "awaits": false, "yields": false }, { "name": "run_native_video", "qualname": "SimpleEngine._stream_chat_impl.run_native_video", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl.run_native_video", "kind": "nested function", "signature": "def run_native_video()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl.run_native_video` calls `mllm_call_kwargs`, `list`, `self._model.stream_chat`; returns `list(self._model.stream_chat(messages=messages, max_tokens=max_tokens, temperature=temperature, tools=template_tools, *…`.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl.run_native_video` calls `mllm_call_kwargs`, `list`, `self._model.stream_chat`; returns `list(self._model.stream_chat(messages=messages, max_tokens=max_tokens, temperature=temperature, tools=template_tools, *…`.", "documented": false, "public": false, "addressable": false, "line": 1299, "end_line": 1309, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1299-L1309", "decorators": [], "calls": [ "mllm_call_kwargs", "list", "self._model.stream_chat" ], "state_reads": [ "self._model.stream_chat", "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "list(self._model.stream_chat(messages=messages, max_tokens=max_tokens, temperature=temperature, tools=template_tools, *…" ], "awaits": false, "yields": false }, { "name": "_to_msg_dict", "qualname": "SimpleEngine._stream_chat_impl._to_msg_dict", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl._to_msg_dict", "kind": "nested function", "signature": "def _to_msg_dict(m: Any) -> dict[str, Any]", "parameters": [ { "name": "m", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl._to_msg_dict` calls `isinstance`, `hasattr`, `m.model_dump`, `m.dict`; has 4 explicit return paths.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl._to_msg_dict` calls `isinstance`, `hasattr`, `m.model_dump`, `m.dict`; has 4 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 1499, "end_line": 1509, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1499-L1509", "decorators": [], "calls": [ "isinstance", "hasattr", "m.model_dump", "m.dict", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "m", "m.model_dump()", "m.dict()", "{'role': getattr(m, 'role', None), 'content': getattr(m, 'content', '')}" ], "awaits": false, "yields": false }, { "name": "_with_user", "qualname": "SimpleEngine._stream_chat_impl._with_user", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl._with_user", "kind": "nested function", "signature": "def _with_user(user_content: str) -> list[dict[str, Any]]", "parameters": [ { "name": "user_content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict[str, Any]]", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl._with_user` calls `dict`, `msgs[-1].get`; returns `msgs`.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl._with_user` calls `dict`, `msgs[-1].get`; returns `msgs`.", "documented": false, "public": false, "addressable": false, "line": 1519, "end_line": 1525, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1519-L1525", "decorators": [], "calls": [ "dict", "msgs[-1].get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "msgs" ], "awaits": false, "yields": false }, { "name": "_emit_response", "qualname": "SimpleEngine._stream_chat_impl._emit_response", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl._emit_response", "kind": "nested function", "signature": "def _emit_response(resp: Any) -> None", "parameters": [ { "name": "resp", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl._emit_response` calls `abort_event.is_set`, `loop.call_soon_threadsafe`; returns `None`.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl._emit_response` calls `abort_event.is_set`, `loop.call_soon_threadsafe`; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 1609, "end_line": 1612, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1609-L1612", "decorators": [], "calls": [ "abort_event.is_set", "loop.call_soon_threadsafe" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_emit_done", "qualname": "SimpleEngine._stream_chat_impl._emit_done", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl._emit_done", "kind": "nested function", "signature": "def _emit_done() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl._emit_done` calls `loop.call_soon_threadsafe`.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl._emit_done` calls `loop.call_soon_threadsafe`.", "documented": false, "public": false, "addressable": false, "line": 1614, "end_line": 1615, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1614-L1615", "decorators": [], "calls": [ "loop.call_soon_threadsafe" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_emit_error", "qualname": "SimpleEngine._stream_chat_impl._emit_error", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl._emit_error", "kind": "nested function", "signature": "def _emit_error(exc: BaseException) -> None", "parameters": [ { "name": "exc", "kind": "positional or keyword", "annotation": "BaseException", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl._emit_error` calls `loop.call_soon_threadsafe`.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl._emit_error` calls `loop.call_soon_threadsafe`.", "documented": false, "public": false, "addressable": false, "line": 1617, "end_line": 1618, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1617-L1618", "decorators": [], "calls": [ "loop.call_soon_threadsafe" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_run_with_cache", "qualname": "SimpleEngine._stream_chat_impl._run_with_cache", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl._run_with_cache", "kind": "nested function", "signature": "def _run_with_cache() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl._run_with_cache` calls `make_sampler`, `make_prompt_cache`, `self._restore_prompt_cache`, `self._system_kv_cache.move_to_end`.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl._run_with_cache` calls `make_sampler`, `make_prompt_cache`, `self._restore_prompt_cache`, `self._system_kv_cache.move_to_end`.", "documented": false, "public": false, "addressable": false, "line": 1620, "end_line": 1705, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1620-L1705", "decorators": [], "calls": [ "make_sampler", "make_prompt_cache", "self._restore_prompt_cache", "self._system_kv_cache.move_to_end", "mx.array", "model", "self._eval_cache_snapshot", "mx.clear_cache", "self._snapshot_prompt_cache", "len", "self._system_kv_cache.popitem", "logger.info", "sum", "mlx_stream_generate", "abort_event.is_set", "_emit_response" ], "state_reads": [ "self._model.model", "self._model", "self._restore_prompt_cache", "self._system_kv_cache", "self._system_kv_cache.move_to_end", "self._system_kv_cache_stats", "self._prefill_step_size", "self._eval_cache_snapshot", "self._snapshot_prompt_cache", "self._system_kv_capacity", "self._system_kv_cache.popitem" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_produce_responses", "qualname": "SimpleEngine._stream_chat_impl._produce_responses", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_chat_impl._produce_responses", "kind": "nested function", "signature": "async def _produce_responses() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_chat_impl._produce_responses` calls `self._run_blocking_serialized`, `_emit_error`, `_emit_done`; awaits asynchronous work.", "implementation": "Nested Function `SimpleEngine._stream_chat_impl._produce_responses` calls `self._run_blocking_serialized`, `_emit_error`, `_emit_done`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 1707, "end_line": 1718, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1707-L1718", "decorators": [], "calls": [ "self._run_blocking_serialized", "_emit_error", "_emit_done" ], "state_reads": [ "self._run_blocking_serialized" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_stream_generate_specprefill", "qualname": "SimpleEngine._stream_generate_specprefill", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_specprefill", "kind": "method", "signature": "async def _stream_generate_specprefill(self, prompt: str, tokens: list[int], max_tokens: int, temperature: float, top_p: float, stop: list[str] | None=None, specprefill_keep_pct: float | None=None, specprefill_backbone_pct: float | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "specprefill_keep_pct", "kind": "positional or keyword", "annotation": "float | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "specprefill_backbone_pct", "kind": "positional or keyword", "annotation": "float | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "SpecPrefill path for non-MTP models (Nemotron, GPT-OSS, etc).\n\nScores token importance with the draft model, sparse-prefills the target\nmodel, then generates autoregressively. Falls back to normal generation\non any error.", "summary": "SpecPrefill path for non-MTP models (Nemotron, GPT-OSS, etc).", "implementation": "Method `SimpleEngine._stream_generate_specprefill` calls `len`, `Event`, `self._run_blocking_serialized`, `enumerate`; awaits asynchronous work; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 1796, "end_line": 2000, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1796-L2000", "decorators": [], "calls": [ "len", "Event", "self._run_blocking_serialized", "enumerate", "GenerationOutput" ], "state_reads": [ "self._model.model", "self._model", "self._model.tokenizer", "self._run_blocking_serialized" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": true }, { "name": "_request_cancel", "qualname": "SimpleEngine._stream_generate_specprefill._request_cancel", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_specprefill._request_cancel", "kind": "nested function", "signature": "def _request_cancel() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_specprefill._request_cancel` calls `cancel_requested.set`.", "implementation": "Nested Function `SimpleEngine._stream_generate_specprefill._request_cancel` calls `cancel_requested.set`.", "documented": false, "public": false, "addressable": false, "line": 1821, "end_line": 1822, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1821-L1822", "decorators": [], "calls": [ "cancel_requested.set" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_cancel_check", "qualname": "SimpleEngine._stream_generate_specprefill._cancel_check", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_specprefill._cancel_check", "kind": "nested function", "signature": "def _cancel_check() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_specprefill._cancel_check` calls `cancel_requested.is_set`, `_SpecPrefillCancelled`; can raise `_SpecPrefillCancelled`.", "implementation": "Nested Function `SimpleEngine._stream_generate_specprefill._cancel_check` calls `cancel_requested.is_set`, `_SpecPrefillCancelled`; can raise `_SpecPrefillCancelled`.", "documented": false, "public": false, "addressable": false, "line": 1824, "end_line": 1826, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1824-L1826", "decorators": [], "calls": [ "cancel_requested.is_set", "_SpecPrefillCancelled" ], "state_reads": [], "state_writes": [], "raises": [ "_SpecPrefillCancelled" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_run_all", "qualname": "SimpleEngine._stream_generate_specprefill._run_all", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_specprefill._run_all", "kind": "nested function", "signature": "def _run_all()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_specprefill._run_all` calls `_run_specprefill`, `logger.error`, `_run_normal`; has 2 explicit return paths.", "implementation": "Nested Function `SimpleEngine._stream_generate_specprefill._run_all` calls `_run_specprefill`, `logger.error`, `_run_normal`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 1828, "end_line": 1835, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1828-L1835", "decorators": [], "calls": [ "_run_specprefill", "logger.error", "_run_normal" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_run_specprefill()", "_run_normal()" ], "awaits": false, "yields": false }, { "name": "_run_specprefill", "qualname": "SimpleEngine._stream_generate_specprefill._run_specprefill", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_specprefill._run_specprefill", "kind": "nested function", "signature": "def _run_specprefill()", "parameters": [], "return_annotation": "not annotated", "docstring": "Score tokens, sparse prefill, generate autoregressively.", "summary": "Score tokens, sparse prefill, generate autoregressively.", "implementation": "Nested Function `SimpleEngine._stream_generate_specprefill._run_specprefill` calls `make_prompt_cache`, `time.monotonic`, `score_tokens`, `_cancel_check`; returns `results`.", "documented": true, "public": false, "addressable": false, "line": 1837, "end_line": 1939, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1837-L1939", "decorators": [], "calls": [ "make_prompt_cache", "time.monotonic", "score_tokens", "_cancel_check", "select_chunks", "sparse_prefill", "logger.info", "make_sampler", "sampler(logits[:, -1, :]).item", "sampler", "tokenizer.decode", "SimpleNamespace", "self._model.stream_generate", "mx.array", "hasattr", "str", "results.append", "getattr", "cleanup_rope" ], "state_reads": [ "self._max_kv_size", "self._draft_model", "self._prefill_step_size", "self._specprefill_keep_pct", "self._specprefill_backbone_pct", "self._model.stream_generate", "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "_run_normal", "qualname": "SimpleEngine._stream_generate_specprefill._run_normal", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_specprefill._run_normal", "kind": "nested function", "signature": "def _run_normal()", "parameters": [], "return_annotation": "not annotated", "docstring": "Fallback: normal generation without specprefill.", "summary": "Fallback: normal generation without specprefill.", "implementation": "Nested Function `SimpleEngine._stream_generate_specprefill._run_normal` calls `self._model.stream_generate`, `_cancel_check`, `hasattr`, `str`; returns `results`.", "documented": true, "public": false, "addressable": false, "line": 1941, "end_line": 1962, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L1941-L1962", "decorators": [], "calls": [ "self._model.stream_generate", "_cancel_check", "hasattr", "str", "results.append", "SimpleNamespace", "getattr" ], "state_reads": [ "self._model.stream_generate", "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "_stream_generate_text", "qualname": "SimpleEngine._stream_generate_text", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text", "kind": "method", "signature": "async def _stream_generate_text(self, messages: list[dict[str, Any]], max_tokens: int, temperature: float, top_p: float, tools: list | None=None, **kwargs) -> AsyncIterator[GenerationOutput]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "AsyncIterator[GenerationOutput]", "docstring": "Text-only generation via mlx_lm TextModel.\n\nUsed when text-only MLLM routing is active and the request has no media.\nRuns the full generation in a single thread to maintain Metal safety.\n\nSystem prompt KV caching: on the first request, prefills system tokens\nand snapshots backbone KV state. Subsequent requests with the same\nsystem prompt restore the snapshot and only prefill the suffix tokens.", "summary": "Text-only generation via mlx_lm TextModel.", "implementation": "Method `SimpleEngine._stream_generate_text` calls `kwargs.pop`, `dict`, `threading.Event`, `os.environ.get`; awaits asynchronous work; yields values incrementally; can raise `payload`.", "documented": true, "public": false, "addressable": true, "line": 2002, "end_line": 2734, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2002-L2734", "decorators": [], "calls": [ "kwargs.pop", "dict", "threading.Event", "os.environ.get", "enable_thinking_env.lower", "template_kwargs.update", "normalize_messages_for_chat_template", "self._text_tokenizer.apply_chat_template", "template_kwargs.pop", "make_sampler", "make_logits_processors", "bool", "cache_blocking_controls.append", "logger.info", "any", "m.get", "full_prompt.find", "hashlib.sha256(system_prefix_text.encode()).hexdigest", "hashlib.sha256", "system_prefix_text.encode", "full_prompt.startswith", "tokenizer.encode", "len", "self._system_kv_cache.get", "self._run_blocking_serialized", "self._system_kv_cache.move_to_end", "logger.debug", "logger.warning", "asyncio.get_running_loop", "asyncio.Queue", "asyncio.create_task", "_produce_responses", "response_queue.get", "hasattr", "str", "getattr", "GenerationOutput", "producer_task.done", "abort_event.set" ], "state_reads": [ "self._text_tokenizer.apply_chat_template", "self._text_tokenizer", "self._supports_system_kv_cache", "self._text_model", "self._system_kv_cache.get", "self._system_kv_cache", "self._run_blocking_serialized", "self._system_kv_cache.move_to_end", "self._system_kv_cache_stats", "self._draft_model", "self._specprefill_threshold" ], "state_writes": [], "raises": [ "payload" ], "return_expressions": [], "awaits": true, "yields": true }, { "name": "make_cache_with_snapshot", "qualname": "SimpleEngine._stream_generate_text.make_cache_with_snapshot", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text.make_cache_with_snapshot", "kind": "nested function", "signature": "def make_cache_with_snapshot(text_model, system_kv_snapshot, _max_kv_size=self._max_kv_size)", "parameters": [ { "name": "text_model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "system_kv_snapshot", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "_max_kv_size", "kind": "positional or keyword", "annotation": "not annotated", "default": "self._max_kv_size", "required": false, "description": "Optional positional or keyword input; defaults to `self._max_kv_size`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text.make_cache_with_snapshot` calls `make_prompt_cache`, `SimpleEngine._restore_prompt_cache`, `mx.array`; returns `(backbone_cache, prompt_to_send)`.", "implementation": "Nested Function `SimpleEngine._stream_generate_text.make_cache_with_snapshot` calls `make_prompt_cache`, `SimpleEngine._restore_prompt_cache`, `mx.array`; returns `(backbone_cache, prompt_to_send)`.", "documented": false, "public": false, "addressable": false, "line": 2159, "end_line": 2176, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2159-L2176", "decorators": [], "calls": [ "make_prompt_cache", "SimpleEngine._restore_prompt_cache", "mx.array" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(backbone_cache, prompt_to_send)" ], "awaits": false, "yields": false }, { "name": "_emit_response", "qualname": "SimpleEngine._stream_generate_text._emit_response", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._emit_response", "kind": "nested function", "signature": "def _emit_response(resp: Any) -> None", "parameters": [ { "name": "resp", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text._emit_response` calls `abort_event.is_set`, `loop.call_soon_threadsafe`; returns `None`.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._emit_response` calls `abort_event.is_set`, `loop.call_soon_threadsafe`; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 2272, "end_line": 2275, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2272-L2275", "decorators": [], "calls": [ "abort_event.is_set", "loop.call_soon_threadsafe" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_emit_done", "qualname": "SimpleEngine._stream_generate_text._emit_done", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._emit_done", "kind": "nested function", "signature": "def _emit_done() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text._emit_done` calls `loop.call_soon_threadsafe`.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._emit_done` calls `loop.call_soon_threadsafe`.", "documented": false, "public": false, "addressable": false, "line": 2277, "end_line": 2278, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2277-L2278", "decorators": [], "calls": [ "loop.call_soon_threadsafe" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_emit_error", "qualname": "SimpleEngine._stream_generate_text._emit_error", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._emit_error", "kind": "nested function", "signature": "def _emit_error(exc: BaseException) -> None", "parameters": [ { "name": "exc", "kind": "positional or keyword", "annotation": "BaseException", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text._emit_error` calls `loop.call_soon_threadsafe`.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._emit_error` calls `loop.call_soon_threadsafe`.", "documented": false, "public": false, "addressable": false, "line": 2280, "end_line": 2281, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2280-L2281", "decorators": [], "calls": [ "loop.call_soon_threadsafe" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_seed_from_last_response", "qualname": "SimpleEngine._stream_generate_text._seed_from_last_response", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._seed_from_last_response", "kind": "nested function", "signature": "def _seed_from_last_response(prompt_cache, last_resp)", "parameters": [ { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "last_resp", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text._seed_from_last_response` calls `getattr`, `cache_module.trim_prompt_cache`, `mx.array`, `self._text_tokenizer.encode`; has 2 explicit return paths.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._seed_from_last_response` calls `getattr`, `cache_module.trim_prompt_cache`, `mx.array`, `self._text_tokenizer.encode`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 2283, "end_line": 2291, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2283-L2291", "decorators": [], "calls": [ "getattr", "cache_module.trim_prompt_cache", "mx.array", "self._text_tokenizer.encode" ], "state_reads": [ "self._text_tokenizer.encode", "self._text_tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "mx.array([last_tok], dtype=mx.uint32)", "mx.array(self._text_tokenizer.encode(getattr(last_resp, 'text', '')), dtype=mx.uint32)" ], "awaits": false, "yields": false }, { "name": "_resume_after_processor_retirement", "qualname": "SimpleEngine._stream_generate_text._resume_after_processor_retirement", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._resume_after_processor_retirement", "kind": "nested function", "signature": "def _resume_after_processor_retirement(model, prompt_cache, prompt, remaining_tokens: int) -> None", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "remaining_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text._resume_after_processor_retirement` calls `dict`, `hasattr`, `model.make_mtp_cache`, `mlx_stream_generate`.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._resume_after_processor_retirement` calls `dict`, `hasattr`, `model.make_mtp_cache`, `mlx_stream_generate`.", "documented": false, "public": false, "addressable": false, "line": 2293, "end_line": 2320, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2293-L2320", "decorators": [], "calls": [ "dict", "hasattr", "model.make_mtp_cache", "mlx_stream_generate", "abort_event.is_set", "logger.info", "_emit_response" ], "state_reads": [ "self._prefill_step_size", "self._mtp_num_draft_tokens", "self._text_tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_run_all", "qualname": "SimpleEngine._stream_generate_text._run_all", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._run_all", "kind": "nested function", "signature": "def _run_all()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text._run_all` calls `_processors_can_retire`, `hasattr`, `logger.info`, `make_prompt_cache`; returns `None`.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._run_all` calls `_processors_can_retire`, `hasattr`, `logger.info`, `make_prompt_cache`; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 2323, "end_line": 2485, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2323-L2485", "decorators": [], "calls": [ "_processors_can_retire", "hasattr", "logger.info", "make_prompt_cache", "mx.array", "model", "self._eval_cache_snapshot", "mx.clear_cache", "self._snapshot_prompt_cache", "self._system_kv_cache.move_to_end", "len", "self._system_kv_cache.popitem", "sum", "_run_specprefill", "logger.error", "model.make_mtp_cache", "dict", "mlx_stream_generate", "abort_event.is_set", "_emit_response", "_processors_retired", "_seed_from_last_response", "_resume_after_processor_retirement" ], "state_reads": [ "self._text_model", "self._mtp", "self._max_kv_size", "self._prefill_step_size", "self._eval_cache_snapshot", "self._snapshot_prompt_cache", "self._system_kv_cache", "self._system_kv_cache.move_to_end", "self._system_kv_capacity", "self._system_kv_cache.popitem", "self._system_kv_cache_stats", "self._mtp_num_draft_tokens", "self._text_tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_run_specprefill", "qualname": "SimpleEngine._stream_generate_text._run_specprefill", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._run_specprefill", "kind": "nested function", "signature": "def _run_specprefill(model, bc, use_mtp)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "bc", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "use_mtp", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Score tokens, sparse prefill, then continue on the standard decode path.", "summary": "Score tokens, sparse prefill, then continue on the standard decode path.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._run_specprefill` calls `make_prompt_cache`, `time.monotonic`, `score_tokens`, `select_chunks`; returns `None`.", "documented": true, "public": false, "addressable": false, "line": 2487, "end_line": 2664, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2487-L2664", "decorators": [], "calls": [ "make_prompt_cache", "time.monotonic", "score_tokens", "select_chunks", "len", "sparse_prefill", "logger.info", "mx.array", "_seed_logits_processors", "_sample_with_processors", "logits[:, -1, :].squeeze", "mx.eval", "y.item", "generated_ids.append", "self._text_tokenizer.decode", "_emit_response", "SimpleNamespace", "abort_event.is_set", "hasattr", "model.make_mtp_cache", "_processors_retired", "_resume_after_processor_retirement", "mlx_stream_generate", "_seed_from_last_response", "cleanup_rope" ], "state_reads": [ "self._max_kv_size", "self._draft_model", "self._prefill_step_size", "self._specprefill_keep_pct", "self._specprefill_backbone_pct", "self._text_tokenizer.eos_token_id", "self._text_tokenizer", "self._text_tokenizer.decode" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_produce_responses", "qualname": "SimpleEngine._stream_generate_text._produce_responses", "full_name": "vllm_mlx.engine.simple.SimpleEngine._stream_generate_text._produce_responses", "kind": "nested function", "signature": "async def _produce_responses() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `SimpleEngine._stream_generate_text._produce_responses` calls `self._run_blocking_serialized`, `_emit_error`, `_emit_done`; awaits asynchronous work.", "implementation": "Nested Function `SimpleEngine._stream_generate_text._produce_responses` calls `self._run_blocking_serialized`, `_emit_error`, `_emit_done`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 2666, "end_line": 2677, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2666-L2677", "decorators": [], "calls": [ "self._run_blocking_serialized", "_emit_error", "_emit_done" ], "state_reads": [ "self._run_blocking_serialized" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "get_stats", "qualname": "SimpleEngine.get_stats", "full_name": "vllm_mlx.engine.simple.SimpleEngine.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict[str, Any]", "parameters": [], "return_annotation": "dict[str, Any]", "docstring": "Get engine statistics.", "summary": "Get engine statistics.", "implementation": "Method `SimpleEngine.get_stats` calls `sum`, `time.time`, `self._active_requests.values`, `dict`; returns `stats`.", "documented": true, "public": true, "addressable": true, "line": 2736, "end_line": 2858, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2736-L2858", "decorators": [], "calls": [ "sum", "time.time", "self._active_requests.values", "dict", "requests_snapshot.append", "self._generation_lock.locked", "self._model.get_cache_stats", "raw_cache.get", "float", "round", "self._system_kv_cache.items", "isinstance", "len", "slots.append", "mx.metal.is_available", "mx.get_active_memory", "mx.get_peak_memory", "mx.get_cache_memory" ], "state_reads": [ "self._recent_completions", "self._active_requests.values", "self._active_requests", "self._model_name", "self._created_at", "self._is_mllm", "self._loaded", "self._num_running", "self._generation_waiters", "self._total_requests_processed", "self._total_prompt_tokens", "self._total_completion_tokens", "self._generation_lock.locked", "self._generation_lock", "self._generation_lock_admission", "self._generation_busy_rejections", "self._model", "self._model.get_cache_stats", "self._draft_model", "self._specprefill_draft_model_path", "self._specprefill_threshold", "self._specprefill_keep_pct", "self._specprefill_backbone_pct", "self._system_kv_cache", "self._system_kv_cache.items", "self._system_kv_cache_stats", "self._system_kv_capacity" ], "state_writes": [], "raises": [], "return_expressions": [ "stats" ], "awaits": false, "yields": false }, { "name": "get_cache_stats", "qualname": "SimpleEngine.get_cache_stats", "full_name": "vllm_mlx.engine.simple.SimpleEngine.get_cache_stats", "kind": "method", "signature": "def get_cache_stats(self) -> dict[str, Any] | None", "parameters": [], "return_annotation": "dict[str, Any] | None", "docstring": "Get cache statistics for the system-prompt KV LRU plus, when the\nmodel is multimodal, the MLLM's own cache stats.", "summary": "Get cache statistics for the system-prompt KV LRU plus, when the model is multimodal, the MLLM's own cache stats.", "implementation": "Method `SimpleEngine.get_cache_stats` calls `dict`, `round`, `len`, `self._model.get_cache_stats`; returns `result or None`.", "documented": true, "public": true, "addressable": true, "line": 2860, "end_line": 2878, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2860-L2878", "decorators": [], "calls": [ "dict", "round", "len", "self._model.get_cache_stats" ], "state_reads": [ "self._supports_system_kv_cache", "self._system_kv_cache_stats", "self._system_kv_capacity", "self._system_kv_cache", "self._is_mllm", "self._model", "self._model.get_cache_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "result or None" ], "awaits": false, "yields": false }, { "name": "clear_runtime_caches", "qualname": "SimpleEngine.clear_runtime_caches", "full_name": "vllm_mlx.engine.simple.SimpleEngine.clear_runtime_caches", "kind": "method", "signature": "def clear_runtime_caches(self) -> dict[str, Any] | None", "parameters": [], "return_annotation": "dict[str, Any] | None", "docstring": "Clear engine-managed runtime caches.\n\nIncludes the multi-slot system-prompt KV LRU — each retained snapshot\nis multi-GB on the Metal heap, so DELETE /v1/cache must drop them or\nthe operator's reset is silently incomplete. Counters reset alongside\nso /v1/cache/stats reflects the cleared state immediately.\n\nOrderedDict ops are atomic under the GIL: a concurrent worker that has\nalready captured a tuple reference from .get() finishes safely against\nits own copy; any new request after this call hits MISS and repopulates\nfrom scratch. No need to acquire _generation_lock for the clear itself.", "summary": "Clear engine-managed runtime caches.", "implementation": "Method `SimpleEngine.clear_runtime_caches` calls `len`, `any`, `self._system_kv_cache_stats.values`, `self._system_kv_cache.clear`; returns `result or None`.", "documented": true, "public": true, "addressable": true, "line": 2880, "end_line": 2912, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine/simple.py#L2880-L2912", "decorators": [], "calls": [ "len", "any", "self._system_kv_cache_stats.values", "self._system_kv_cache.clear", "mx.clear_cache", "self._model.clear_cache" ], "state_reads": [ "self._system_kv_cache", "self._system_kv_cache_stats.values", "self._system_kv_cache_stats", "self._system_kv_cache.clear", "self._is_mllm", "self._model", "self._model.clear_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "result or None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.engine_core", "path": "vllm_mlx/engine_core.py", "page_path": "reference/api/vllm_mlx/engine_core.md", "docstring": "Engine Core for vllm-mlx continuous batching.\n\nThis module provides the EngineCore class that coordinates:\n- Model loading and management\n- Request scheduling via Scheduler\n- Async request processing\n- Output streaming\n\nThe design follows vLLM's engine architecture adapted for MLX.", "summary": "Engine Core for vllm-mlx continuous batching.", "line_count": 794, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L1-L794", "members": [ "logger", "_is_stream_thread_error", "EngineConfig", "EngineCore", "AsyncEngineCore" ], "symbols": [ { "name": "_is_stream_thread_error", "qualname": "_is_stream_thread_error", "full_name": "vllm_mlx.engine_core._is_stream_thread_error", "kind": "function", "signature": "def _is_stream_thread_error(error: Exception) -> bool", "parameters": [ { "name": "error", "kind": "positional or keyword", "annotation": "Exception", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "True when MLX reports stream ownership mismatch across threads.", "summary": "True when MLX reports stream ownership mismatch across threads.", "implementation": "Function `_is_stream_thread_error` calls `str`; returns `'no Stream(' in message or 'no Stream(gpu' in message`.", "documented": true, "public": false, "addressable": true, "line": 33, "end_line": 36, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L33-L36", "decorators": [], "calls": [ "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'no Stream(' in message or 'no Stream(gpu' in message" ], "awaits": false, "yields": false }, { "name": "EngineConfig", "qualname": "EngineConfig", "full_name": "vllm_mlx.engine_core.EngineConfig", "kind": "class", "signature": "class EngineConfig", "parameters": [ { "name": "model_name", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "scheduler_config", "kind": "field", "annotation": "Optional[SchedulerConfig]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "step_interval", "kind": "field", "annotation": "float", "default": "0.001", "required": false, "description": "Optional constructor field; defaults to `0.001`." }, { "name": "stream_interval", "kind": "field", "annotation": "int", "default": "1", "required": false, "description": "Optional constructor field; defaults to `1`." }, { "name": "gpu_memory_utilization", "kind": "field", "annotation": "float", "default": "0.9", "required": false, "description": "Optional constructor field; defaults to `0.9`." } ], "return_annotation": "EngineConfig", "docstring": "Configuration for the engine.", "summary": "Configuration for the engine.", "implementation": "Class `EngineConfig` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 47, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L40-L47", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "EngineCore", "qualname": "EngineCore", "full_name": "vllm_mlx.engine_core.EngineCore", "kind": "class", "signature": "class EngineCore", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model" }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The tokenizer" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[EngineConfig]", "default": "None", "required": false, "description": "Engine configuration" }, { "name": "engine_id", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional unique ID for this engine (auto-generated if None)" }, { "name": "force_model_ownership", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True (default), forcibly take model ownership from any existing engine. If False, raises ModelOwnershipError if model is in use." } ], "return_annotation": "EngineCore", "docstring": "Core engine for vllm-mlx inference with continuous batching.\n\nThis engine runs the generation loop and manages request lifecycle.\nIt provides both sync and async interfaces for request handling.", "summary": "Core engine for vllm-mlx inference with continuous batching.", "implementation": "Class `EngineCore` declares 21 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 50, "end_line": 698, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L50-L698", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "EngineCore.__init__", "full_name": "vllm_mlx.engine_core.EngineCore.__init__", "kind": "method", "signature": "def __init__(self, model: Any, tokenizer: Any, config: Optional[EngineConfig]=None, engine_id: Optional[str]=None, force_model_ownership: bool=True)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model" }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The tokenizer" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[EngineConfig]", "default": "None", "required": false, "description": "Engine configuration" }, { "name": "engine_id", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional unique ID for this engine (auto-generated if None)" }, { "name": "force_model_ownership", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True (default), forcibly take model ownership from any existing engine. If False, raises ModelOwnershipError if model is in use." } ], "return_annotation": "not annotated", "docstring": "Initialize the engine.\n\nArgs:\n model: The MLX model\n tokenizer: The tokenizer\n config: Engine configuration\n engine_id: Optional unique ID for this engine (auto-generated if None)\n force_model_ownership: If True (default), forcibly take model ownership\n from any existing engine. If False, raises\n ModelOwnershipError if model is in use.", "summary": "Initialize the engine.", "implementation": "Method `EngineCore.__init__` updates `self.model`, `self.tokenizer`, `self.config`, `self._engine_id`; calls `EngineConfig`, `str`, `uuid.uuid4`, `get_registry`.", "documented": true, "public": false, "addressable": true, "line": 58, "end_line": 114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L58-L114", "decorators": [], "calls": [ "EngineConfig", "str", "uuid.uuid4", "get_registry", "registry.acquire", "SchedulerConfig", "Scheduler", "logger.debug" ], "state_reads": [ "self._engine_id", "self.config.scheduler_config", "self.config" ], "state_writes": [ "self.model", "self.tokenizer", "self.config", "self._engine_id", "self._owns_model", "self._closed", "self.scheduler", "self._output_collectors", "self._stream_states", "self._finished_events", "self._running", "self._task", "self._start_time", "self._steps_executed" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start", "qualname": "EngineCore.start", "full_name": "vllm_mlx.engine_core.EngineCore.start", "kind": "method", "signature": "async def start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the engine loop.", "summary": "Start the engine loop.", "implementation": "Method `EngineCore.start` updates `self._running`, `self._start_time`, `self._task`; calls `time.time`, `asyncio.create_task`, `self._engine_loop`, `logger.info`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 116, "end_line": 124, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L116-L124", "decorators": [], "calls": [ "time.time", "asyncio.create_task", "self._engine_loop", "logger.info" ], "state_reads": [ "self._running", "self._engine_loop" ], "state_writes": [ "self._running", "self._start_time", "self._task" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "stop", "qualname": "EngineCore.stop", "full_name": "vllm_mlx.engine_core.EngineCore.stop", "kind": "method", "signature": "async def stop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop the engine loop.", "summary": "Stop the engine loop.", "implementation": "Method `EngineCore.stop` updates `self._running`, `self._task`; calls `self._task.cancel`, `self.scheduler._close_batch_generator`, `logger.info`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 126, "end_line": 140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L126-L140", "decorators": [], "calls": [ "self._task.cancel", "self.scheduler._close_batch_generator", "logger.info" ], "state_reads": [ "self._task", "self._task.cancel", "self.scheduler._close_batch_generator", "self.scheduler" ], "state_writes": [ "self._running", "self._task" ], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "is_running", "qualname": "EngineCore.is_running", "full_name": "vllm_mlx.engine_core.EngineCore.is_running", "kind": "method", "signature": "def is_running(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if engine is running.", "summary": "Check if engine is running.", "implementation": "Method `EngineCore.is_running` returns `self._running`.", "documented": true, "public": true, "addressable": true, "line": 142, "end_line": 144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L142-L144", "decorators": [], "calls": [], "state_reads": [ "self._running" ], "state_writes": [], "raises": [], "return_expressions": [ "self._running" ], "awaits": false, "yields": false }, { "name": "_engine_loop", "qualname": "EngineCore._engine_loop", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop", "kind": "method", "signature": "async def _engine_loop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Main engine loop.\n\nscheduler.step runs on one dedicated worker thread. MLX streams are\nthread-local, so we rebind generation streams inside that worker.", "summary": "Main engine loop.", "implementation": "Method `EngineCore._engine_loop` calls `asyncio.get_running_loop`, `ThreadPoolExecutor`, `mx.device_info().get`, `mx.device_info`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 146, "end_line": 334, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L146-L334", "decorators": [], "calls": [ "asyncio.get_running_loop", "ThreadPoolExecutor", "mx.device_info().get", "mx.device_info", "int", "min", "self.scheduler.has_requests", "loop.run_in_executor", "_is_stream_thread_error", "_bind_model_streams_once", "logger.warning", "_step_on_model_thread", "asyncio.sleep", "collectors.get", "collector.put", "states.get", "state.should_send", "state.mark_sent", "events.get", "event.set", "mx.clear_cache", "logger.error", "traceback.format_exc", "self.scheduler._close_batch_generator", "worker.shutdown" ], "state_reads": [ "self.config.step_interval", "self.config", "self.config.stream_interval", "self.config.gpu_memory_utilization", "self._running", "self.scheduler.has_requests", "self.scheduler", "self._output_collectors", "self._stream_states", "self._finished_events", "self.scheduler._close_batch_generator" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_bind_worker_streams_once", "qualname": "EngineCore._engine_loop._bind_worker_streams_once", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop._bind_worker_streams_once", "kind": "nested function", "signature": "def _bind_worker_streams_once() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `EngineCore._engine_loop._bind_worker_streams_once` calls `bind_generation_streams`.", "implementation": "Nested Function `EngineCore._engine_loop._bind_worker_streams_once` calls `bind_generation_streams`.", "documented": false, "public": false, "addressable": false, "line": 160, "end_line": 164, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L160-L164", "decorators": [], "calls": [ "bind_generation_streams" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_bind_model_streams_once", "qualname": "EngineCore._engine_loop._bind_model_streams_once", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop._bind_model_streams_once", "kind": "nested function", "signature": "def _bind_model_streams_once() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `EngineCore._engine_loop._bind_model_streams_once` calls `bind_generation_streams`.", "implementation": "Nested Function `EngineCore._engine_loop._bind_model_streams_once` calls `bind_generation_streams`.", "documented": false, "public": false, "addressable": false, "line": 166, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L166-L170", "decorators": [], "calls": [ "bind_generation_streams" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_step_on_worker", "qualname": "EngineCore._engine_loop._step_on_worker", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop._step_on_worker", "kind": "nested function", "signature": "def _step_on_worker()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `EngineCore._engine_loop._step_on_worker` updates `self._steps_executed`; calls `_bind_worker_streams_once`, `self.scheduler.step`, `mx.get_active_memory`, `mx.clear_cache`; returns `output`.", "implementation": "Nested Function `EngineCore._engine_loop._step_on_worker` updates `self._steps_executed`; calls `_bind_worker_streams_once`, `self.scheduler.step`, `mx.get_active_memory`, `mx.clear_cache`; returns `output`.", "documented": false, "public": false, "addressable": false, "line": 172, "end_line": 190, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L172-L190", "decorators": [], "calls": [ "_bind_worker_streams_once", "self.scheduler.step", "mx.get_active_memory", "mx.clear_cache", "logger.warning" ], "state_reads": [ "self.scheduler.step", "self.scheduler", "self._steps_executed" ], "state_writes": [ "self._steps_executed" ], "raises": [], "return_expressions": [ "output" ], "awaits": false, "yields": false }, { "name": "_step_on_model_thread", "qualname": "EngineCore._engine_loop._step_on_model_thread", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop._step_on_model_thread", "kind": "nested function", "signature": "def _step_on_model_thread()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `EngineCore._engine_loop._step_on_model_thread` updates `self._steps_executed`; calls `_bind_model_streams_once`, `self.scheduler.step`, `mx.get_active_memory`, `mx.clear_cache`; returns `output`.", "implementation": "Nested Function `EngineCore._engine_loop._step_on_model_thread` updates `self._steps_executed`; calls `_bind_model_streams_once`, `self.scheduler.step`, `mx.get_active_memory`, `mx.clear_cache`; returns `output`.", "documented": false, "public": false, "addressable": false, "line": 192, "end_line": 210, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L192-L210", "decorators": [], "calls": [ "_bind_model_streams_once", "self.scheduler.step", "mx.get_active_memory", "mx.clear_cache", "logger.warning" ], "state_reads": [ "self.scheduler.step", "self.scheduler", "self._steps_executed" ], "state_writes": [ "self._steps_executed" ], "raises": [], "return_expressions": [ "output" ], "awaits": false, "yields": false }, { "name": "_recover_stream_thread_error_on_worker", "qualname": "EngineCore._engine_loop._recover_stream_thread_error_on_worker", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop._recover_stream_thread_error_on_worker", "kind": "nested function", "signature": "def _recover_stream_thread_error_on_worker() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `EngineCore._engine_loop._recover_stream_thread_error_on_worker` calls `_bind_worker_streams_once`, `self.scheduler._recover_from_cache_error`, `self.scheduler._reschedule_running_requests`.", "implementation": "Nested Function `EngineCore._engine_loop._recover_stream_thread_error_on_worker` calls `_bind_worker_streams_once`, `self.scheduler._recover_from_cache_error`, `self.scheduler._reschedule_running_requests`.", "documented": false, "public": false, "addressable": false, "line": 212, "end_line": 215, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L212-L215", "decorators": [], "calls": [ "_bind_worker_streams_once", "self.scheduler._recover_from_cache_error", "self.scheduler._reschedule_running_requests" ], "state_reads": [ "self.scheduler._recover_from_cache_error", "self.scheduler", "self.scheduler._reschedule_running_requests" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_clear_cache_on_worker", "qualname": "EngineCore._engine_loop._clear_cache_on_worker", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop._clear_cache_on_worker", "kind": "nested function", "signature": "def _clear_cache_on_worker() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `EngineCore._engine_loop._clear_cache_on_worker` calls `_bind_worker_streams_once`, `mx.clear_cache`.", "implementation": "Nested Function `EngineCore._engine_loop._clear_cache_on_worker` calls `_bind_worker_streams_once`, `mx.clear_cache`.", "documented": false, "public": false, "addressable": false, "line": 217, "end_line": 219, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L217-L219", "decorators": [], "calls": [ "_bind_worker_streams_once", "mx.clear_cache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_close_batch_generator_on_worker", "qualname": "EngineCore._engine_loop._close_batch_generator_on_worker", "full_name": "vllm_mlx.engine_core.EngineCore._engine_loop._close_batch_generator_on_worker", "kind": "nested function", "signature": "def _close_batch_generator_on_worker() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `EngineCore._engine_loop._close_batch_generator_on_worker` calls `_bind_worker_streams_once`, `self.scheduler._close_batch_generator`.", "implementation": "Nested Function `EngineCore._engine_loop._close_batch_generator_on_worker` calls `_bind_worker_streams_once`, `self.scheduler._close_batch_generator`.", "documented": false, "public": false, "addressable": false, "line": 221, "end_line": 223, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L221-L223", "decorators": [], "calls": [ "_bind_worker_streams_once", "self.scheduler._close_batch_generator" ], "state_reads": [ "self.scheduler._close_batch_generator", "self.scheduler" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "add_request", "qualname": "EngineCore.add_request", "full_name": "vllm_mlx.engine_core.EngineCore.add_request", "kind": "method", "signature": "async def add_request(self, prompt: Union[str, List[int]], sampling_params: Optional[SamplingParams]=None, request_id: Optional[str]=None, images: Optional[List[Any]]=None, videos: Optional[List[Any]]=None, prefix_boundary: int=0) -> str", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "Union[str, List[int]]", "default": "", "required": true, "description": "Input prompt (string or token IDs)" }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "Optional[SamplingParams]", "default": "None", "required": false, "description": "Generation parameters" }, { "name": "request_id", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional custom request ID" }, { "name": "images", "kind": "positional or keyword", "annotation": "Optional[List[Any]]", "default": "None", "required": false, "description": "Optional images for multimodal" }, { "name": "videos", "kind": "positional or keyword", "annotation": "Optional[List[Any]]", "default": "None", "required": false, "description": "Optional videos for multimodal" }, { "name": "prefix_boundary", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Token count for shared prefix (for cache)" } ], "return_annotation": "str", "docstring": "Add a request for processing.\n\nArgs:\n prompt: Input prompt (string or token IDs)\n sampling_params: Generation parameters\n request_id: Optional custom request ID\n images: Optional images for multimodal\n videos: Optional videos for multimodal\n prefix_boundary: Token count for shared prefix (for cache)\n\nReturns:\n The request ID", "summary": "Add a request for processing.", "implementation": "Method `EngineCore.add_request` calls `str`, `uuid.uuid4`, `SamplingParams`, `Request`; returns `request_id`.", "documented": true, "public": true, "addressable": true, "line": 336, "end_line": 384, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L336-L384", "decorators": [], "calls": [ "str", "uuid.uuid4", "SamplingParams", "Request", "RequestOutputCollector", "RequestStreamState", "asyncio.Event", "self.scheduler.add_request" ], "state_reads": [ "self._output_collectors", "self._stream_states", "self.config.stream_interval", "self.config", "self._finished_events", "self.scheduler.add_request", "self.scheduler" ], "state_writes": [], "raises": [], "return_expressions": [ "request_id" ], "awaits": false, "yields": false }, { "name": "abort_request", "qualname": "EngineCore.abort_request", "full_name": "vllm_mlx.engine_core.EngineCore.abort_request", "kind": "method", "signature": "async def abort_request(self, request_id: str) -> bool", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Abort a request.", "summary": "Abort a request.", "implementation": "Method `EngineCore.abort_request` calls `self.scheduler.abort_request`, `self._cleanup_request`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 386, "end_line": 390, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L386-L390", "decorators": [], "calls": [ "self.scheduler.abort_request", "self._cleanup_request" ], "state_reads": [ "self.scheduler.abort_request", "self.scheduler", "self._cleanup_request" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "_cleanup_request", "qualname": "EngineCore._cleanup_request", "full_name": "vllm_mlx.engine_core.EngineCore._cleanup_request", "kind": "method", "signature": "def _cleanup_request(self, request_id: str) -> None", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Clean up request tracking.", "summary": "Clean up request tracking.", "implementation": "Method `EngineCore._cleanup_request` calls `self._output_collectors.pop`, `collector.clear`, `self._stream_states.pop`, `self._finished_events.pop`.", "documented": true, "public": false, "addressable": true, "line": 392, "end_line": 399, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L392-L399", "decorators": [], "calls": [ "self._output_collectors.pop", "collector.clear", "self._stream_states.pop", "self._finished_events.pop", "self.scheduler.remove_finished_request" ], "state_reads": [ "self._output_collectors.pop", "self._output_collectors", "self._stream_states.pop", "self._stream_states", "self._finished_events.pop", "self._finished_events", "self.scheduler.remove_finished_request", "self.scheduler" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "stream_outputs", "qualname": "EngineCore.stream_outputs", "full_name": "vllm_mlx.engine_core.EngineCore.stream_outputs", "kind": "method", "signature": "async def stream_outputs(self, request_id: str, timeout: Optional[float]=None) -> AsyncIterator[RequestOutput]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The request ID" }, { "name": "timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional timeout in seconds" } ], "return_annotation": "AsyncIterator[RequestOutput]", "docstring": "Stream outputs for a request with low-latency non-blocking pattern.\n\nUses the vLLM pattern: get_nowait() or await get()\nThis avoids unnecessary task switches when output is available.\n\nArgs:\n request_id: The request ID\n timeout: Optional timeout in seconds\n\nYields:\n RequestOutput objects as tokens are generated", "summary": "Stream outputs for a request with low-latency non-blocking pattern.", "implementation": "Method `EngineCore.stream_outputs` calls `_time.monotonic`, `self._output_collectors.get`, `logger.warning`, `logger.info`; awaits asynchronous work; yields values incrementally; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 401, "end_line": 488, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L401-L488", "decorators": [], "calls": [ "_time.monotonic", "self._output_collectors.get", "logger.warning", "logger.info", "collector.get_nowait", "asyncio.wait_for", "collector.get", "type", "self.scheduler.abort_request", "self._cleanup_request" ], "state_reads": [ "self._output_collectors.get", "self._output_collectors", "self.scheduler.abort_request", "self.scheduler", "self._cleanup_request" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": true }, { "name": "generate", "qualname": "EngineCore.generate", "full_name": "vllm_mlx.engine_core.EngineCore.generate", "kind": "method", "signature": "async def generate(self, prompt: Union[str, List[int]], sampling_params: Optional[SamplingParams]=None, request_id: Optional[str]=None, **kwargs) -> RequestOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "Union[str, List[int]]", "default": "", "required": true, "description": "Input prompt" }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "Optional[SamplingParams]", "default": "None", "required": false, "description": "Generation parameters" }, { "name": "request_id", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional request ID" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "RequestOutput", "docstring": "Generate a complete response (non-streaming).\n\nThis method is optimized to avoid streaming overhead when\nyou only need the final result.\n\nArgs:\n prompt: Input prompt\n sampling_params: Generation parameters\n request_id: Optional request ID\n\nReturns:\n Final RequestOutput with complete text", "summary": "Generate a complete response (non-streaming).", "implementation": "Method `EngineCore.generate` calls `self.add_request`, `self._finished_events.get`, `RuntimeError`, `event.wait`; awaits asynchronous work; can raise `RuntimeError`; returns `final_output`.", "documented": true, "public": true, "addressable": true, "line": 490, "end_line": 552, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L490-L552", "decorators": [], "calls": [ "self.add_request", "self._finished_events.get", "RuntimeError", "event.wait", "self._output_collectors.get", "collector.get_nowait", "logger.info", "self.scheduler.abort_request", "self._cleanup_request" ], "state_reads": [ "self.add_request", "self._finished_events.get", "self._finished_events", "self._output_collectors.get", "self._output_collectors", "self.scheduler.abort_request", "self.scheduler", "self._cleanup_request" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "final_output" ], "awaits": true, "yields": false }, { "name": "generate_batch_sync", "qualname": "EngineCore.generate_batch_sync", "full_name": "vllm_mlx.engine_core.EngineCore.generate_batch_sync", "kind": "method", "signature": "def generate_batch_sync(self, prompts: List[Union[str, List[int]]], sampling_params: Optional[SamplingParams]=None) -> List[RequestOutput]", "parameters": [ { "name": "prompts", "kind": "positional or keyword", "annotation": "List[Union[str, List[int]]]", "default": "", "required": true, "description": "List of input prompts" }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "Optional[SamplingParams]", "default": "None", "required": false, "description": "Generation parameters (same for all)" } ], "return_annotation": "List[RequestOutput]", "docstring": "Generate responses synchronously for maximum throughput.\n\nThis bypasses the async engine loop entirely, running the scheduler\ndirectly for optimal batching performance. Use this when you don't\nneed streaming and want maximum throughput.\n\nArgs:\n prompts: List of input prompts\n sampling_params: Generation parameters (same for all)\n\nReturns:\n List of RequestOutput in same order as prompts", "summary": "Generate responses synchronously for maximum throughput.", "implementation": "Method `EngineCore.generate_batch_sync` calls `SamplingParams`, `str`, `uuid_module.uuid4`, `Request`; returns `[results[rid] for rid in request_ids]`.", "documented": true, "public": true, "addressable": true, "line": 554, "end_line": 609, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L554-L609", "decorators": [], "calls": [ "SamplingParams", "str", "uuid_module.uuid4", "Request", "self.scheduler.add_request", "request_ids.append", "bind_generation_streams", "self.scheduler.has_requests", "self.scheduler.step", "self.scheduler.remove_finished_request" ], "state_reads": [ "self.scheduler.add_request", "self.scheduler", "self.scheduler.has_requests", "self.scheduler.step", "self.scheduler.remove_finished_request" ], "state_writes": [], "raises": [], "return_expressions": [ "[results[rid] for rid in request_ids]" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "EngineCore.get_stats", "full_name": "vllm_mlx.engine_core.EngineCore.get_stats", "kind": "method", "signature": "def get_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get engine statistics.", "summary": "Get engine statistics.", "implementation": "Method `EngineCore.get_stats` calls `self.scheduler.get_stats`, `time.time`, `len`, `self.scheduler.get_running_requests_info`; returns `{'running': self._running, 'uptime_seconds': uptime, 'steps_executed': self._steps_executed, 'active_requests': len(sel…`.", "documented": true, "public": true, "addressable": true, "line": 611, "end_line": 624, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L611-L624", "decorators": [], "calls": [ "self.scheduler.get_stats", "time.time", "len", "self.scheduler.get_running_requests_info" ], "state_reads": [ "self.scheduler.get_stats", "self.scheduler", "self._start_time", "self._running", "self._steps_executed", "self._output_collectors", "self.config.stream_interval", "self.config", "self.scheduler.get_running_requests_info" ], "state_writes": [], "raises": [], "return_expressions": [ "{'running': self._running, 'uptime_seconds': uptime, 'steps_executed': self._steps_executed, 'active_requests': len(sel…" ], "awaits": false, "yields": false }, { "name": "get_cache_stats", "qualname": "EngineCore.get_cache_stats", "full_name": "vllm_mlx.engine_core.EngineCore.get_cache_stats", "kind": "method", "signature": "def get_cache_stats(self) -> Optional[Dict[str, Any]]", "parameters": [], "return_annotation": "Optional[Dict[str, Any]]", "docstring": "Get prefix cache statistics.", "summary": "Get prefix cache statistics.", "implementation": "Method `EngineCore.get_cache_stats` calls `self.scheduler.get_cache_stats`; returns `self.scheduler.get_cache_stats()`.", "documented": true, "public": true, "addressable": true, "line": 626, "end_line": 628, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L626-L628", "decorators": [], "calls": [ "self.scheduler.get_cache_stats" ], "state_reads": [ "self.scheduler.get_cache_stats", "self.scheduler" ], "state_writes": [], "raises": [], "return_expressions": [ "self.scheduler.get_cache_stats()" ], "awaits": false, "yields": false }, { "name": "save_cache_to_disk", "qualname": "EngineCore.save_cache_to_disk", "full_name": "vllm_mlx.engine_core.EngineCore.save_cache_to_disk", "kind": "method", "signature": "def save_cache_to_disk(self, cache_dir: str) -> bool", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Save prefix cache to disk.", "summary": "Save prefix cache to disk.", "implementation": "Method `EngineCore.save_cache_to_disk` calls `self.scheduler.save_cache_to_disk`; returns `self.scheduler.save_cache_to_disk(cache_dir)`.", "documented": true, "public": true, "addressable": true, "line": 630, "end_line": 632, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L630-L632", "decorators": [], "calls": [ "self.scheduler.save_cache_to_disk" ], "state_reads": [ "self.scheduler.save_cache_to_disk", "self.scheduler" ], "state_writes": [], "raises": [], "return_expressions": [ "self.scheduler.save_cache_to_disk(cache_dir)" ], "awaits": false, "yields": false }, { "name": "load_cache_from_disk", "qualname": "EngineCore.load_cache_from_disk", "full_name": "vllm_mlx.engine_core.EngineCore.load_cache_from_disk", "kind": "method", "signature": "def load_cache_from_disk(self, cache_dir: str) -> int", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Load prefix cache from disk.", "summary": "Load prefix cache from disk.", "implementation": "Method `EngineCore.load_cache_from_disk` calls `self.scheduler.load_cache_from_disk`; returns `self.scheduler.load_cache_from_disk(cache_dir)`.", "documented": true, "public": true, "addressable": true, "line": 634, "end_line": 636, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L634-L636", "decorators": [], "calls": [ "self.scheduler.load_cache_from_disk" ], "state_reads": [ "self.scheduler.load_cache_from_disk", "self.scheduler" ], "state_writes": [], "raises": [], "return_expressions": [ "self.scheduler.load_cache_from_disk(cache_dir)" ], "awaits": false, "yields": false }, { "name": "clear_runtime_caches", "qualname": "EngineCore.clear_runtime_caches", "full_name": "vllm_mlx.engine_core.EngineCore.clear_runtime_caches", "kind": "method", "signature": "def clear_runtime_caches(self) -> Dict[str, Any] | None", "parameters": [], "return_annotation": "Dict[str, Any] | None", "docstring": "Clear scheduler-managed runtime caches.", "summary": "Clear scheduler-managed runtime caches.", "implementation": "Method `EngineCore.clear_runtime_caches` calls `self.scheduler.clear_runtime_caches`; returns `self.scheduler.clear_runtime_caches()`.", "documented": true, "public": true, "addressable": true, "line": 638, "end_line": 640, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L638-L640", "decorators": [], "calls": [ "self.scheduler.clear_runtime_caches" ], "state_reads": [ "self.scheduler.clear_runtime_caches", "self.scheduler" ], "state_writes": [], "raises": [], "return_expressions": [ "self.scheduler.clear_runtime_caches()" ], "awaits": false, "yields": false }, { "name": "clear_prefix_cache", "qualname": "EngineCore.clear_prefix_cache", "full_name": "vllm_mlx.engine_core.EngineCore.clear_prefix_cache", "kind": "method", "signature": "def clear_prefix_cache(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear the prefix cache (delegates to scheduler).", "summary": "Clear the prefix cache (delegates to scheduler).", "implementation": "Method `EngineCore.clear_prefix_cache` calls `hasattr`, `self.scheduler.clear_prefix_cache`.", "documented": true, "public": true, "addressable": true, "line": 642, "end_line": 645, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L642-L645", "decorators": [], "calls": [ "hasattr", "self.scheduler.clear_prefix_cache" ], "state_reads": [ "self.scheduler", "self.scheduler.clear_prefix_cache" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_release_model", "qualname": "EngineCore._release_model", "full_name": "vllm_mlx.engine_core.EngineCore._release_model", "kind": "method", "signature": "def _release_model(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Release model ownership.", "summary": "Release model ownership.", "implementation": "Method `EngineCore._release_model` updates `self._owns_model`; calls `get_registry`, `registry.release`, `logger.debug`.", "documented": true, "public": false, "addressable": true, "line": 647, "end_line": 653, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L647-L653", "decorators": [], "calls": [ "get_registry", "registry.release", "logger.debug" ], "state_reads": [ "self._owns_model", "self._closed", "self.model", "self._engine_id" ], "state_writes": [ "self._owns_model" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "close", "qualname": "EngineCore.close", "full_name": "vllm_mlx.engine_core.EngineCore.close", "kind": "method", "signature": "def close(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Explicitly close the engine and release resources.\n\nThis should be called when done using the engine, especially\nif you plan to create another engine with the same model.", "summary": "Explicitly close the engine and release resources.", "implementation": "Method `EngineCore.close` updates `self._owns_model`, `self._closed`; calls `get_registry`, `registry.release`, `logger.debug`, `self.scheduler.deep_reset`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 655, "end_line": 685, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L655-L685", "decorators": [], "calls": [ "get_registry", "registry.release", "logger.debug", "self.scheduler.deep_reset", "self._output_collectors.values", "collector.clear", "self._output_collectors.clear", "self._stream_states.clear", "self._finished_events.clear" ], "state_reads": [ "self._closed", "self._owns_model", "self.model", "self._engine_id", "self.scheduler.deep_reset", "self.scheduler", "self._output_collectors.values", "self._output_collectors", "self._output_collectors.clear", "self._stream_states.clear", "self._stream_states", "self._finished_events.clear", "self._finished_events" ], "state_writes": [ "self._owns_model", "self._closed" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "__del__", "qualname": "EngineCore.__del__", "full_name": "vllm_mlx.engine_core.EngineCore.__del__", "kind": "method", "signature": "def __del__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Cleanup on destruction.", "summary": "Cleanup on destruction.", "implementation": "Method `EngineCore.__del__` calls `self._release_model`.", "documented": true, "public": false, "addressable": true, "line": 687, "end_line": 693, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L687-L693", "decorators": [], "calls": [ "self._release_model" ], "state_reads": [ "self._release_model" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "engine_id", "qualname": "EngineCore.engine_id", "full_name": "vllm_mlx.engine_core.EngineCore.engine_id", "kind": "method", "signature": "def engine_id(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Get the engine ID.", "summary": "Get the engine ID.", "implementation": "Method `EngineCore.engine_id` returns `self._engine_id`.", "documented": true, "public": true, "addressable": true, "line": 696, "end_line": 698, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L696-L698", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._engine_id" ], "state_writes": [], "raises": [], "return_expressions": [ "self._engine_id" ], "awaits": false, "yields": false }, { "name": "AsyncEngineCore", "qualname": "AsyncEngineCore", "full_name": "vllm_mlx.engine_core.AsyncEngineCore", "kind": "class", "signature": "class AsyncEngineCore", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[EngineConfig]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "AsyncEngineCore", "docstring": "Async context manager wrapper for EngineCore.\n\nUsage:\n async with AsyncEngineCore(model, tokenizer) as engine:\n request_id = await engine.add_request(\"Hello\")\n async for output in engine.stream_outputs(request_id):\n print(output.new_text)", "summary": "Async context manager wrapper for EngineCore.", "implementation": "Class `AsyncEngineCore` declares 14 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 701, "end_line": 794, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L701-L794", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "AsyncEngineCore.__init__", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.__init__", "kind": "method", "signature": "def __init__(self, model: Any, tokenizer: Any, config: Optional[EngineConfig]=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[EngineConfig]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `AsyncEngineCore.__init__` updates `self.engine`; calls `EngineCore`.", "implementation": "Method `AsyncEngineCore.__init__` updates `self.engine`; calls `EngineCore`.", "documented": false, "public": false, "addressable": true, "line": 712, "end_line": 718, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L712-L718", "decorators": [], "calls": [ "EngineCore" ], "state_reads": [], "state_writes": [ "self.engine" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__aenter__", "qualname": "AsyncEngineCore.__aenter__", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.__aenter__", "kind": "method", "signature": "async def __aenter__(self) -> 'AsyncEngineCore'", "parameters": [], "return_annotation": "'AsyncEngineCore'", "docstring": "", "summary": "Method `AsyncEngineCore.__aenter__` calls `self.engine.start`; awaits asynchronous work; returns `self`.", "implementation": "Method `AsyncEngineCore.__aenter__` calls `self.engine.start`; awaits asynchronous work; returns `self`.", "documented": false, "public": false, "addressable": true, "line": 720, "end_line": 722, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L720-L722", "decorators": [], "calls": [ "self.engine.start" ], "state_reads": [ "self.engine.start", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "self" ], "awaits": true, "yields": false }, { "name": "__aexit__", "qualname": "AsyncEngineCore.__aexit__", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.__aexit__", "kind": "method", "signature": "async def __aexit__(self, *args) -> None", "parameters": [ { "name": "*args", "kind": "variadic positional", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic positional inputs accepted by this callable." } ], "return_annotation": "None", "docstring": "", "summary": "Method `AsyncEngineCore.__aexit__` calls `self.engine.stop`; awaits asynchronous work.", "implementation": "Method `AsyncEngineCore.__aexit__` calls `self.engine.stop`; awaits asynchronous work.", "documented": false, "public": false, "addressable": true, "line": 724, "end_line": 725, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L724-L725", "decorators": [], "calls": [ "self.engine.stop" ], "state_reads": [ "self.engine.stop", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "start", "qualname": "AsyncEngineCore.start", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.start", "kind": "method", "signature": "def start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start engine (creates task in current loop).", "summary": "Start engine (creates task in current loop).", "implementation": "Method `AsyncEngineCore.start` updates `self._start_task`; calls `asyncio.create_task`, `self.engine.start`.", "documented": true, "public": true, "addressable": true, "line": 727, "end_line": 729, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L727-L729", "decorators": [], "calls": [ "asyncio.create_task", "self.engine.start" ], "state_reads": [ "self.engine.start", "self.engine" ], "state_writes": [ "self._start_task" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "stop", "qualname": "AsyncEngineCore.stop", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.stop", "kind": "method", "signature": "async def stop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop the engine.", "summary": "Stop the engine.", "implementation": "Method `AsyncEngineCore.stop` calls `self.engine.stop`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 731, "end_line": 733, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L731-L733", "decorators": [], "calls": [ "self.engine.stop" ], "state_reads": [ "self.engine.stop", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "add_request", "qualname": "AsyncEngineCore.add_request", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.add_request", "kind": "method", "signature": "async def add_request(self, prompt: Union[str, List[int]], sampling_params: Optional[SamplingParams]=None, request_id: Optional[str]=None, **kwargs) -> str", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "Union[str, List[int]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "Optional[SamplingParams]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request_id", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "str", "docstring": "Add a request.", "summary": "Add a request.", "implementation": "Method `AsyncEngineCore.add_request` calls `self.engine.add_request`; awaits asynchronous work; returns `await self.engine.add_request(prompt=prompt, sampling_params=sampling_params, request_id=request_id, **kwargs)`.", "documented": true, "public": true, "addressable": true, "line": 735, "end_line": 748, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L735-L748", "decorators": [], "calls": [ "self.engine.add_request" ], "state_reads": [ "self.engine.add_request", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "await self.engine.add_request(prompt=prompt, sampling_params=sampling_params, request_id=request_id, **kwargs)" ], "awaits": true, "yields": false }, { "name": "abort_request", "qualname": "AsyncEngineCore.abort_request", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.abort_request", "kind": "method", "signature": "async def abort_request(self, request_id: str) -> bool", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Abort a request.", "summary": "Abort a request.", "implementation": "Method `AsyncEngineCore.abort_request` calls `self.engine.abort_request`; awaits asynchronous work; returns `await self.engine.abort_request(request_id)`.", "documented": true, "public": true, "addressable": true, "line": 750, "end_line": 752, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L750-L752", "decorators": [], "calls": [ "self.engine.abort_request" ], "state_reads": [ "self.engine.abort_request", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "await self.engine.abort_request(request_id)" ], "awaits": true, "yields": false }, { "name": "stream_outputs", "qualname": "AsyncEngineCore.stream_outputs", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.stream_outputs", "kind": "method", "signature": "async def stream_outputs(self, request_id: str, timeout: Optional[float]=None) -> AsyncIterator[RequestOutput]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "AsyncIterator[RequestOutput]", "docstring": "Stream outputs.", "summary": "Stream outputs.", "implementation": "Method `AsyncEngineCore.stream_outputs` calls `self.engine.stream_outputs`; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 754, "end_line": 761, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L754-L761", "decorators": [], "calls": [ "self.engine.stream_outputs" ], "state_reads": [ "self.engine.stream_outputs", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "generate", "qualname": "AsyncEngineCore.generate", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.generate", "kind": "method", "signature": "async def generate(self, prompt: Union[str, List[int]], sampling_params: Optional[SamplingParams]=None, **kwargs) -> RequestOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "Union[str, List[int]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "Optional[SamplingParams]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "RequestOutput", "docstring": "Generate complete response.", "summary": "Generate complete response.", "implementation": "Method `AsyncEngineCore.generate` calls `self.engine.generate`; awaits asynchronous work; returns `await self.engine.generate(prompt=prompt, sampling_params=sampling_params, **kwargs)`.", "documented": true, "public": true, "addressable": true, "line": 763, "end_line": 774, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L763-L774", "decorators": [], "calls": [ "self.engine.generate" ], "state_reads": [ "self.engine.generate", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "await self.engine.generate(prompt=prompt, sampling_params=sampling_params, **kwargs)" ], "awaits": true, "yields": false }, { "name": "get_stats", "qualname": "AsyncEngineCore.get_stats", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.get_stats", "kind": "method", "signature": "def get_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get engine stats.", "summary": "Get engine stats.", "implementation": "Method `AsyncEngineCore.get_stats` calls `self.engine.get_stats`; returns `self.engine.get_stats()`.", "documented": true, "public": true, "addressable": true, "line": 776, "end_line": 778, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L776-L778", "decorators": [], "calls": [ "self.engine.get_stats" ], "state_reads": [ "self.engine.get_stats", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "self.engine.get_stats()" ], "awaits": false, "yields": false }, { "name": "get_cache_stats", "qualname": "AsyncEngineCore.get_cache_stats", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.get_cache_stats", "kind": "method", "signature": "def get_cache_stats(self) -> Optional[Dict[str, Any]]", "parameters": [], "return_annotation": "Optional[Dict[str, Any]]", "docstring": "Get prefix cache statistics.", "summary": "Get prefix cache statistics.", "implementation": "Method `AsyncEngineCore.get_cache_stats` calls `self.engine.get_cache_stats`; returns `self.engine.get_cache_stats()`.", "documented": true, "public": true, "addressable": true, "line": 780, "end_line": 782, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L780-L782", "decorators": [], "calls": [ "self.engine.get_cache_stats" ], "state_reads": [ "self.engine.get_cache_stats", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "self.engine.get_cache_stats()" ], "awaits": false, "yields": false }, { "name": "save_cache_to_disk", "qualname": "AsyncEngineCore.save_cache_to_disk", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.save_cache_to_disk", "kind": "method", "signature": "def save_cache_to_disk(self, cache_dir: str) -> bool", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Save prefix cache to disk.", "summary": "Save prefix cache to disk.", "implementation": "Method `AsyncEngineCore.save_cache_to_disk` calls `self.engine.save_cache_to_disk`; returns `self.engine.save_cache_to_disk(cache_dir)`.", "documented": true, "public": true, "addressable": true, "line": 784, "end_line": 786, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L784-L786", "decorators": [], "calls": [ "self.engine.save_cache_to_disk" ], "state_reads": [ "self.engine.save_cache_to_disk", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "self.engine.save_cache_to_disk(cache_dir)" ], "awaits": false, "yields": false }, { "name": "load_cache_from_disk", "qualname": "AsyncEngineCore.load_cache_from_disk", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.load_cache_from_disk", "kind": "method", "signature": "def load_cache_from_disk(self, cache_dir: str) -> int", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Load prefix cache from disk.", "summary": "Load prefix cache from disk.", "implementation": "Method `AsyncEngineCore.load_cache_from_disk` calls `self.engine.load_cache_from_disk`; returns `self.engine.load_cache_from_disk(cache_dir)`.", "documented": true, "public": true, "addressable": true, "line": 788, "end_line": 790, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L788-L790", "decorators": [], "calls": [ "self.engine.load_cache_from_disk" ], "state_reads": [ "self.engine.load_cache_from_disk", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "self.engine.load_cache_from_disk(cache_dir)" ], "awaits": false, "yields": false }, { "name": "clear_runtime_caches", "qualname": "AsyncEngineCore.clear_runtime_caches", "full_name": "vllm_mlx.engine_core.AsyncEngineCore.clear_runtime_caches", "kind": "method", "signature": "def clear_runtime_caches(self) -> Dict[str, Any] | None", "parameters": [], "return_annotation": "Dict[str, Any] | None", "docstring": "Clear scheduler-managed runtime caches.", "summary": "Clear scheduler-managed runtime caches.", "implementation": "Method `AsyncEngineCore.clear_runtime_caches` calls `self.engine.clear_runtime_caches`; returns `self.engine.clear_runtime_caches()`.", "documented": true, "public": true, "addressable": true, "line": 792, "end_line": 794, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/engine_core.py#L792-L794", "decorators": [], "calls": [ "self.engine.clear_runtime_caches" ], "state_reads": [ "self.engine.clear_runtime_caches", "self.engine" ], "state_writes": [], "raises": [], "return_expressions": [ "self.engine.clear_runtime_caches()" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.gradio_app", "path": "vllm_mlx/gradio_app.py", "page_path": "reference/api/vllm_mlx/gradio_app.md", "docstring": "Gradio Chatbot Interface for vllm-mlx.\n\nA multimodal chat interface that connects to the vllm-mlx server\nand supports text, images, and video files.\n\nUsage:\n # First start the server with a multimodal model:\n vllm-mlx serve --served-model-name default mlx-community/Qwen3-VL-4B-Instruct-3bit --port 8000\n\n # Then run the app:\n vllm-mlx-chat\n\n # Or with a different served-model name served on localhost:8000:\n vllm-mlx-chat --served-model-name --server-url http://localhost:8000 --port 7860\n\nNote:\n Query the /v1/models endpoint on localhost with `curl` and `jq` to see available models and their names:\n ```bash\n curl http://localhost:8000/v1/models | jq \".data[0].id\"\n ```", "summary": "Gradio Chatbot Interface for vllm-mlx.", "line_count": 411, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_app.py#L1-L411", "members": [ "encode_file_to_base64", "build_message_content", "create_chat_function", "main" ], "symbols": [ { "name": "encode_file_to_base64", "qualname": "encode_file_to_base64", "full_name": "vllm_mlx.gradio_app.encode_file_to_base64", "kind": "function", "signature": "def encode_file_to_base64(file_path: str) -> tuple[str, str]", "parameters": [ { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, str]", "docstring": "Encode a file to base64 data URL.\n\nReturns:\n Tuple of (data_url, media_type) where media_type is 'image' or 'video'", "summary": "Encode a file to base64 data URL.", "implementation": "Function `encode_file_to_base64` calls `Path`, `path.suffix.lower`, `open`, `base64.b64encode(f.read()).decode`; returns `(f'data:{mime_type};base64,{data}', media_type)`.", "documented": true, "public": true, "addressable": true, "line": 33, "end_line": 76, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_app.py#L33-L76", "decorators": [], "calls": [ "Path", "path.suffix.lower", "open", "base64.b64encode(f.read()).decode", "base64.b64encode", "f.read" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(f'data:{mime_type};base64,{data}', media_type)" ], "awaits": false, "yields": false }, { "name": "build_message_content", "qualname": "build_message_content", "full_name": "vllm_mlx.gradio_app.build_message_content", "kind": "function", "signature": "def build_message_content(text: str, files: list[str] | None=None) -> list | str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The text message" }, { "name": "files", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional list of file paths (images or videos)" } ], "return_annotation": "list | str", "docstring": "Build OpenAI-compatible message content with text and optional files.\n\nArgs:\n text: The text message\n files: Optional list of file paths (images or videos)\n\nReturns:\n Content in OpenAI multimodal format", "summary": "Build OpenAI-compatible message content with text and optional files.", "implementation": "Function `build_message_content` calls `content.append`, `encode_file_to_base64`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 79, "end_line": 108, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_app.py#L79-L108", "decorators": [], "calls": [ "content.append", "encode_file_to_base64" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text", "content if content else text" ], "awaits": false, "yields": false }, { "name": "create_chat_function", "qualname": "create_chat_function", "full_name": "vllm_mlx.gradio_app.create_chat_function", "kind": "function", "signature": "def create_chat_function(server_url: str, max_tokens: int, temperature: float, served_model_name: str='default')", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "URL of the vllm-mlx server" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Sampling temperature" }, { "name": "served_model_name", "kind": "positional or keyword", "annotation": "str", "default": "'default'", "required": false, "description": "Model name to send in OpenAI-compatible requests" } ], "return_annotation": "not annotated", "docstring": "Create the chat function for Gradio ChatInterface.\n\nArgs:\n server_url: URL of the vllm-mlx server\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n served_model_name: Model name to send in OpenAI-compatible requests\n\nReturns:\n Chat function compatible with gr.ChatInterface", "summary": "Create the chat function for Gradio ChatInterface.", "implementation": "Function `create_chat_function` returns `chat`.", "documented": true, "public": true, "addressable": true, "line": 111, "end_line": 257, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_app.py#L111-L257", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "chat" ], "awaits": false, "yields": false }, { "name": "chat", "qualname": "create_chat_function.chat", "full_name": "vllm_mlx.gradio_app.create_chat_function.chat", "kind": "nested function", "signature": "def chat(message: dict, history: list) -> str", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Dict with 'text' and optional 'files' keys" }, { "name": "history", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "List of previous messages" } ], "return_annotation": "str", "docstring": "Process a multimodal message and return response.\n\nArgs:\n message: Dict with 'text' and optional 'files' keys\n history: List of previous messages\n\nReturns:\n Assistant response text", "summary": "Process a multimodal message and return response.", "implementation": "Nested Function `create_chat_function.chat` calls `isinstance`, `message.get`, `print`, `len`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 132, "end_line": 255, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_app.py#L132-L255", "decorators": [], "calls": [ "isinstance", "message.get", "print", "len", "sys.stdout.flush", "enumerate", "msg.get", "p.get", "' '.join", "str", "rebuilt_content.append", "messages.append", "content.get", "build_message_content", "encode_file_to_base64", "media_items.append", "c.get", "requests.post", "response.raise_for_status", "response.json" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result['choices'][0]['message']['content']", "'Error: Cannot connect to server. Make sure vllm-mlx is running.'", "'Error: Timeout - server took too long to respond.'", "f'Error: {str(e)}'" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "vllm_mlx.gradio_app.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "Run the Gradio app.", "summary": "Run the Gradio app.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "documented": true, "public": true, "addressable": true, "line": 260, "end_line": 407, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_app.py#L260-L407", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "gr.ChatInterface", "create_chat_function", "gr.MultimodalTextbox", "demo.launch" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "text_chat", "qualname": "main.text_chat", "full_name": "vllm_mlx.gradio_app.main.text_chat", "kind": "nested function", "signature": "def text_chat(message: str, history: list) -> str", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "history", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Process a text-only message.", "summary": "Process a text-only message.", "implementation": "Nested Function `main.text_chat` calls `isinstance`, `msg.get`, `p.get`, `' '.join`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 331, "end_line": 368, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_app.py#L331-L368", "decorators": [], "calls": [ "isinstance", "msg.get", "p.get", "' '.join", "messages.append", "requests.post", "response.raise_for_status", "response.json", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result['choices'][0]['message']['content']", "'Error: Cannot connect to server. Make sure vllm-mlx is running.'", "'Error: Timeout - server took too long to respond.'", "f'Error: {str(e)}'" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.gradio_text_app", "path": "vllm_mlx/gradio_text_app.py", "page_path": "reference/api/vllm_mlx/gradio_text_app.md", "docstring": "Gradio Text-Only Chatbot Interface for vllm-mlx.\n\nA fast, text-only chat interface for LLM models.\nUse this for text conversations without image/video overhead.\n\nUsage:\n # First start the server with a model:\n # Without a custom API model name (model path is the name used in the OpenAI API):\n vllm-mlx serve mlx-community/Llama-3.2-3B-Instruct-4bit --port 8000\n # With a custom API model name (\"default\" is the name used in the OpenAI API):\n vllm-mlx serve --served-model-name default mlx-community/Llama-3.2-3B-Instruct-4bit --port 8000\n\n # Then run this app:\n vllm-mlx-text-chat --served-model-name mlx-community/Llama-3.2-3B-Instruct-4bit\n\n # Or with vllm-mlx started with served model name is 'default', there is no need to use --served-model-name:\n vllm-mlx-text-chat --server-url http://localhost:8000 --port 7861\n\nNote:\n Query the /v1/models endpoint with `curl` and `jq` to see available models and their names:\n ```bash\n curl http://localhost:8000/v1/models | jq \".data[0].id\"\n ```", "summary": "Gradio Text-Only Chatbot Interface for vllm-mlx.", "line_count": 205, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_text_app.py#L1-L205", "members": [ "create_chat_function", "main" ], "symbols": [ { "name": "create_chat_function", "qualname": "create_chat_function", "full_name": "vllm_mlx.gradio_text_app.create_chat_function", "kind": "function", "signature": "def create_chat_function(server_url: str, max_tokens: int, temperature: float, served_model_name: str='default')", "parameters": [ { "name": "server_url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "URL of the vllm-mlx server" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Sampling temperature" }, { "name": "served_model_name", "kind": "positional or keyword", "annotation": "str", "default": "'default'", "required": false, "description": "Model name to send in OpenAI-compatible requests" } ], "return_annotation": "not annotated", "docstring": "Create the chat function for Gradio ChatInterface.\n\nArgs:\n server_url: URL of the vllm-mlx server\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n served_model_name: Model name to send in OpenAI-compatible requests\n\nReturns:\n Chat function compatible with gr.ChatInterface", "summary": "Create the chat function for Gradio ChatInterface.", "implementation": "Function `create_chat_function` returns `chat`.", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 108, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_text_app.py#L34-L108", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "chat" ], "awaits": false, "yields": false }, { "name": "chat", "qualname": "create_chat_function.chat", "full_name": "vllm_mlx.gradio_text_app.create_chat_function.chat", "kind": "nested function", "signature": "def chat(message: str, history: list) -> str", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "User's text message" }, { "name": "history", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "List of previous messages" } ], "return_annotation": "str", "docstring": "Process a text message and return response.\n\nArgs:\n message: User's text message\n history: List of previous messages\n\nReturns:\n Assistant response text", "summary": "Process a text message and return response.", "implementation": "Nested Function `create_chat_function.chat` calls `isinstance`, `msg.get`, `p.get`, `' '.join`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 53, "end_line": 106, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_text_app.py#L53-L106", "decorators": [], "calls": [ "isinstance", "msg.get", "p.get", "' '.join", "messages.append", "requests.post", "response.raise_for_status", "response.json", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result['choices'][0]['message']['content']", "'Error: Cannot connect to server. Make sure vllm-mlx is running.'", "'Error: Timeout - server took too long to respond.'", "f'Error: {str(e)}'" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "vllm_mlx.gradio_text_app.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "Run the Gradio app.", "summary": "Run the Gradio app.", "implementation": "Function `main` calls `argparse.ArgumentParser`, `parser.add_argument`, `parser.parse_args`, `print`.", "documented": true, "public": true, "addressable": true, "line": 111, "end_line": 201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/gradio_text_app.py#L111-L201", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "parser.parse_args", "print", "create_chat_function", "gr.ChatInterface", "demo.launch" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.lifecycle", "path": "vllm_mlx/lifecycle.py", "page_path": "reference/api/vllm_mlx/lifecycle.md", "docstring": "Model lifecycle / residency management for vllm-mlx.", "summary": "Model lifecycle / residency management for vllm-mlx.", "line_count": 493, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L1-L493", "members": [ "ResidentState", "ModelSpec", "ResidentModel", "ResidencyManager" ], "symbols": [ { "name": "ResidentState", "qualname": "ResidentState", "full_name": "vllm_mlx.lifecycle.ResidentState", "kind": "class", "signature": "class ResidentState(str, Enum)", "parameters": [], "return_annotation": "ResidentState", "docstring": "Runtime residency state for a configured model.", "summary": "Runtime residency state for a configured model.", "implementation": "Class `ResidentState` derives from `str`, `Enum` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 17, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L17-L24", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ModelSpec", "qualname": "ModelSpec", "full_name": "vllm_mlx.lifecycle.ModelSpec", "kind": "class", "signature": "class ModelSpec", "parameters": [ { "name": "model_key", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "model_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "use_batching", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "scheduler_config", "kind": "field", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "stream_interval", "kind": "field", "annotation": "int", "default": "1", "required": false, "description": "Optional constructor field; defaults to `1`." }, { "name": "max_tokens", "kind": "field", "annotation": "int", "default": "32768", "required": false, "description": "Optional constructor field; defaults to `32768`." }, { "name": "force_mllm", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "mtp", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "prefill_step_size", "kind": "field", "annotation": "int", "default": "2048", "required": false, "description": "Optional constructor field; defaults to `2048`." }, { "name": "specprefill_enabled", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "specprefill_threshold", "kind": "field", "annotation": "int", "default": "8192", "required": false, "description": "Optional constructor field; defaults to `8192`." }, { "name": "specprefill_keep_pct", "kind": "field", "annotation": "float", "default": "0.3", "required": false, "description": "Optional constructor field; defaults to `0.3`." }, { "name": "specprefill_backbone_pct", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "specprefill_draft_model", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ModelSpec", "docstring": "Immutable engine construction inputs for a resident model.", "summary": "Immutable engine construction inputs for a resident model.", "implementation": "Class `ModelSpec` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 44, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L28-L44", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResidentModel", "qualname": "ResidentModel", "full_name": "vllm_mlx.lifecycle.ResidentModel", "kind": "class", "signature": "class ResidentModel", "parameters": [ { "name": "spec", "kind": "field", "annotation": "ModelSpec", "default": "", "required": true, "description": "Required constructor field." }, { "name": "state", "kind": "field", "annotation": "ResidentState", "default": "ResidentState.UNLOADED", "required": false, "description": "Optional constructor field; defaults to `ResidentState.UNLOADED`." }, { "name": "engine", "kind": "field", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "active_requests", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "last_used_at", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "loaded_at", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "last_error", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "estimated_memory_bytes", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "_load_waiters", "kind": "field", "annotation": "int", "default": "field(default=0, repr=False)", "required": false, "description": "Optional constructor field; defaults to `field(default=0, repr=False)`." }, { "name": "_load_waiter_task", "kind": "field", "annotation": "asyncio.Task[BaseEngine] | None", "default": "field(default=None, repr=False)", "required": false, "description": "Optional constructor field; defaults to `field(default=None, repr=False)`." }, { "name": "_prepare_task", "kind": "field", "annotation": "asyncio.Task[None] | None", "default": "field(default=None, repr=False)", "required": false, "description": "Optional constructor field; defaults to `field(default=None, repr=False)`." }, { "name": "_abandoned_loading_task", "kind": "field", "annotation": "asyncio.Task[BaseEngine] | None", "default": "field(default=None, repr=False)", "required": false, "description": "Optional constructor field; defaults to `field(default=None, repr=False)`." }, { "name": "_loading_task", "kind": "field", "annotation": "asyncio.Task[BaseEngine] | None", "default": "field(default=None, repr=False)", "required": false, "description": "Optional constructor field; defaults to `field(default=None, repr=False)`." }, { "name": "_unloading_task", "kind": "field", "annotation": "asyncio.Task[bool] | None", "default": "field(default=None, repr=False)", "required": false, "description": "Optional constructor field; defaults to `field(default=None, repr=False)`." } ], "return_annotation": "ResidentModel", "docstring": "Runtime state for a single resident model.", "summary": "Runtime state for a single resident model.", "implementation": "Class `ResidentModel` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 48, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L48-L66", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResidencyManager", "qualname": "ResidencyManager", "full_name": "vllm_mlx.lifecycle.ResidencyManager", "kind": "class", "signature": "class ResidencyManager", "parameters": [ { "name": "engine_factory", "kind": "positional or keyword", "annotation": "Callable[[ModelSpec], Awaitable[BaseEngine]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "on_engine_loaded", "kind": "keyword-only", "annotation": "Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "on_engine_unloading", "kind": "keyword-only", "annotation": "Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "time_fn", "kind": "keyword-only", "annotation": "Callable[[], float] | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "auto_unload_idle_seconds", "kind": "keyword-only", "annotation": "float", "default": "0", "required": false, "description": "Optional keyword-only input; defaults to `0`." } ], "return_annotation": "ResidencyManager", "docstring": "Single-flight lifecycle manager for resident models.", "summary": "Single-flight lifecycle manager for resident models.", "implementation": "Class `ResidencyManager` declares 16 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 493, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L69-L493", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ResidencyManager.__init__", "full_name": "vllm_mlx.lifecycle.ResidencyManager.__init__", "kind": "method", "signature": "def __init__(self, engine_factory: Callable[[ModelSpec], Awaitable[BaseEngine]], *, on_engine_loaded: Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None=None, on_engine_unloading: Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None=None, time_fn: Callable[[], float] | None=None, auto_unload_idle_seconds: float=0) -> None", "parameters": [ { "name": "engine_factory", "kind": "positional or keyword", "annotation": "Callable[[ModelSpec], Awaitable[BaseEngine]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "on_engine_loaded", "kind": "keyword-only", "annotation": "Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "on_engine_unloading", "kind": "keyword-only", "annotation": "Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "time_fn", "kind": "keyword-only", "annotation": "Callable[[], float] | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "auto_unload_idle_seconds", "kind": "keyword-only", "annotation": "float", "default": "0", "required": false, "description": "Optional keyword-only input; defaults to `0`." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ResidencyManager.__init__` updates `self._engine_factory`, `self._on_engine_loaded`, `self._on_engine_unloading`, `self._time_fn`; calls `__import__`, `asyncio.Lock`.", "implementation": "Method `ResidencyManager.__init__` updates `self._engine_factory`, `self._on_engine_loaded`, `self._on_engine_unloading`, `self._time_fn`; calls `__import__`, `asyncio.Lock`.", "documented": false, "public": false, "addressable": true, "line": 72, "end_line": 91, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L72-L91", "decorators": [], "calls": [ "__import__", "asyncio.Lock" ], "state_reads": [], "state_writes": [ "self._engine_factory", "self._on_engine_loaded", "self._on_engine_unloading", "self._time_fn", "self.auto_unload_idle_seconds", "self._residents", "self._lock" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "register_model", "qualname": "ResidencyManager.register_model", "full_name": "vllm_mlx.lifecycle.ResidencyManager.register_model", "kind": "method", "signature": "def register_model(self, spec: ModelSpec) -> str", "parameters": [ { "name": "spec", "kind": "positional or keyword", "annotation": "ModelSpec", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Register a model spec, or replace a dormant resident entry.", "summary": "Register a model spec, or replace a dormant resident entry.", "implementation": "Method `ResidencyManager.register_model` calls `self._residents.get`, `RuntimeError`, `ResidentModel`; can raise `RuntimeError`; returns `spec.model_key`.", "documented": true, "public": true, "addressable": true, "line": 93, "end_line": 111, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L93-L111", "decorators": [], "calls": [ "self._residents.get", "RuntimeError", "ResidentModel" ], "state_reads": [ "self._residents.get", "self._residents" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "spec.model_key" ], "awaits": false, "yields": false }, { "name": "get_engine", "qualname": "ResidencyManager.get_engine", "full_name": "vllm_mlx.lifecycle.ResidencyManager.get_engine", "kind": "method", "signature": "def get_engine(self, model_key: str) -> BaseEngine | None", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BaseEngine | None", "docstring": "Get the currently loaded engine, if any.", "summary": "Get the currently loaded engine, if any.", "implementation": "Method `ResidencyManager.get_engine` calls `self._resident`; returns `self._resident(model_key).engine`.", "documented": true, "public": true, "addressable": true, "line": 113, "end_line": 115, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L113-L115", "decorators": [], "calls": [ "self._resident" ], "state_reads": [ "self._resident" ], "state_writes": [], "raises": [], "return_expressions": [ "self._resident(model_key).engine" ], "awaits": false, "yields": false }, { "name": "get_status", "qualname": "ResidencyManager.get_status", "full_name": "vllm_mlx.lifecycle.ResidencyManager.get_status", "kind": "method", "signature": "def get_status(self, model_key: str) -> dict[str, Any]", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Return a serializable snapshot of resident state.", "summary": "Return a serializable snapshot of resident state.", "implementation": "Method `ResidencyManager.get_status` calls `self._resident`; returns `{'model_key': resident.spec.model_key, 'model_name': resident.spec.model_name, 'state': resident.state.value, 'active_r…`.", "documented": true, "public": true, "addressable": true, "line": 117, "end_line": 130, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L117-L130", "decorators": [], "calls": [ "self._resident" ], "state_reads": [ "self._resident", "self.auto_unload_idle_seconds" ], "state_writes": [], "raises": [], "return_expressions": [ "{'model_key': resident.spec.model_key, 'model_name': resident.spec.model_name, 'state': resident.state.value, 'active_r…" ], "awaits": false, "yields": false }, { "name": "ensure_loaded", "qualname": "ResidencyManager.ensure_loaded", "full_name": "vllm_mlx.lifecycle.ResidencyManager.ensure_loaded", "kind": "method", "signature": "async def ensure_loaded(self, model_key: str) -> BaseEngine", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BaseEngine", "docstring": "Load and start a resident engine if needed.", "summary": "Load and start a resident engine if needed.", "implementation": "Method `ResidencyManager.ensure_loaded` calls `self._resident`, `asyncio.create_task`, `self._load_engine`, `asyncio.shield`; awaits asynchronous work; can raise `RuntimeError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 132, "end_line": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L132-L184", "decorators": [], "calls": [ "self._resident", "asyncio.create_task", "self._load_engine", "asyncio.shield", "RuntimeError", "asyncio.current_task", "getattr", "task.done", "task.cancelled", "cancelling", "self._release_load_waiter" ], "state_reads": [ "self._lock", "self._resident", "self._load_engine", "self._release_load_waiter" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "resident.engine", "await asyncio.shield(task)" ], "awaits": true, "yields": false }, { "name": "acquire", "qualname": "ResidencyManager.acquire", "full_name": "vllm_mlx.lifecycle.ResidencyManager.acquire", "kind": "method", "signature": "async def acquire(self, model_key: str, *, count_activity: bool=True) -> BaseEngine", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "count_activity", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "BaseEngine", "docstring": "Acquire a resident engine for request processing.", "summary": "Acquire a resident engine for request processing.", "implementation": "Method `ResidencyManager.acquire` calls `self.ensure_loaded`, `self._resident`, `self._time_fn`; awaits asynchronous work; returns `engine`.", "documented": true, "public": true, "addressable": true, "line": 186, "end_line": 206, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L186-L206", "decorators": [], "calls": [ "self.ensure_loaded", "self._resident", "self._time_fn" ], "state_reads": [ "self.ensure_loaded", "self._lock", "self._resident", "self._time_fn" ], "state_writes": [], "raises": [], "return_expressions": [ "engine" ], "awaits": true, "yields": false }, { "name": "release", "qualname": "ResidencyManager.release", "full_name": "vllm_mlx.lifecycle.ResidencyManager.release", "kind": "method", "signature": "async def release(self, model_key: str, *, count_activity: bool=True) -> None", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "count_activity", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "None", "docstring": "Release a previously acquired resident engine.", "summary": "Release a previously acquired resident engine.", "implementation": "Method `ResidencyManager.release` calls `self._resident`, `self._time_fn`.", "documented": true, "public": true, "addressable": true, "line": 208, "end_line": 215, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L208-L215", "decorators": [], "calls": [ "self._resident", "self._time_fn" ], "state_reads": [ "self._lock", "self._resident", "self._time_fn" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "unload_if_idle", "qualname": "ResidencyManager.unload_if_idle", "full_name": "vllm_mlx.lifecycle.ResidencyManager.unload_if_idle", "kind": "method", "signature": "async def unload_if_idle(self, model_key: str) -> bool", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Unload a resident engine if it has been idle past the threshold.", "summary": "Unload a resident engine if it has been idle past the threshold.", "implementation": "Method `ResidencyManager.unload_if_idle` calls `self._resident`, `self._time_fn`, `asyncio.create_task`, `self._unload_engine`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 217, "end_line": 253, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L217-L253", "decorators": [], "calls": [ "self._resident", "self._time_fn", "asyncio.create_task", "self._unload_engine", "asyncio.shield" ], "state_reads": [ "self.auto_unload_idle_seconds", "self._lock", "self._resident", "self._time_fn", "self._unload_engine" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "await asyncio.shield(unloading_task)" ], "awaits": true, "yields": false }, { "name": "shutdown", "qualname": "ResidencyManager.shutdown", "full_name": "vllm_mlx.lifecycle.ResidencyManager.shutdown", "kind": "method", "signature": "async def shutdown(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop all loaded residents.", "summary": "Stop all loaded residents.", "implementation": "Method `ResidencyManager.shutdown` calls `list`, `self._residents.keys`, `self._resident`, `resident._loading_task.cancel`; awaits asynchronous work; can raise `RuntimeError`.", "documented": true, "public": true, "addressable": true, "line": 255, "end_line": 312, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L255-L312", "decorators": [], "calls": [ "list", "self._residents.keys", "self._resident", "resident._loading_task.cancel", "asyncio.create_task", "self._unload_engine", "suppress", "asyncio.shield", "suspend_cancellation", "failures.append", "len", "RuntimeError", "'; '.join" ], "state_reads": [ "self._residents.keys", "self._residents", "self._lock", "self._resident", "self._unload_engine" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_load_engine", "qualname": "ResidencyManager._load_engine", "full_name": "vllm_mlx.lifecycle.ResidencyManager._load_engine", "kind": "method", "signature": "async def _load_engine(self, resident: ResidentModel) -> BaseEngine", "parameters": [ { "name": "resident", "kind": "positional or keyword", "annotation": "ResidentModel", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BaseEngine", "docstring": "Create and start a resident engine.", "summary": "Create and start a resident engine.", "implementation": "Method `ResidencyManager._load_engine` calls `self._engine_factory`, `self._prepare_engine_start`, `engine.start`, `self._run_hook`; awaits asynchronous work; can raise `asyncio.CancelledError`; returns `engine`.", "documented": true, "public": false, "addressable": true, "line": 314, "end_line": 354, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L314-L354", "decorators": [], "calls": [ "self._engine_factory", "self._prepare_engine_start", "engine.start", "self._run_hook", "self._cleanup_cancelled_load", "asyncio.current_task", "asyncio.CancelledError", "suppress", "engine.stop", "str", "self._time_fn" ], "state_reads": [ "self._engine_factory", "self._prepare_engine_start", "self._run_hook", "self._on_engine_loaded", "self._cleanup_cancelled_load", "self._lock", "self._time_fn" ], "state_writes": [], "raises": [ "asyncio.CancelledError" ], "return_expressions": [ "engine" ], "awaits": true, "yields": false }, { "name": "_unload_engine", "qualname": "ResidencyManager._unload_engine", "full_name": "vllm_mlx.lifecycle.ResidencyManager._unload_engine", "kind": "method", "signature": "async def _unload_engine(self, resident: ResidentModel) -> bool", "parameters": [ { "name": "resident", "kind": "positional or keyword", "annotation": "ResidentModel", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Stop and drop a resident engine.", "summary": "Stop and drop a resident engine.", "implementation": "Method `ResidencyManager._unload_engine` calls `self._run_hook`, `engine.stop`, `str`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 356, "end_line": 388, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L356-L388", "decorators": [], "calls": [ "self._run_hook", "engine.stop", "str" ], "state_reads": [ "self._lock", "self._run_hook", "self._on_engine_unloading" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": true, "yields": false }, { "name": "_resident", "qualname": "ResidencyManager._resident", "full_name": "vllm_mlx.lifecycle.ResidencyManager._resident", "kind": "method", "signature": "def _resident(self, model_key: str) -> ResidentModel", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "ResidentModel", "docstring": "", "summary": "Method `ResidencyManager._resident` calls `KeyError`; can raise `KeyError`; returns `self._residents[model_key]`.", "implementation": "Method `ResidencyManager._resident` calls `KeyError`; can raise `KeyError`; returns `self._residents[model_key]`.", "documented": false, "public": false, "addressable": true, "line": 390, "end_line": 394, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L390-L394", "decorators": [], "calls": [ "KeyError" ], "state_reads": [ "self._residents" ], "state_writes": [], "raises": [ "KeyError" ], "return_expressions": [ "self._residents[model_key]" ], "awaits": false, "yields": false }, { "name": "_run_hook", "qualname": "ResidencyManager._run_hook", "full_name": "vllm_mlx.lifecycle.ResidencyManager._run_hook", "kind": "method", "signature": "async def _run_hook(self, hook: Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None, spec: ModelSpec, engine: BaseEngine) -> None", "parameters": [ { "name": "hook", "kind": "positional or keyword", "annotation": "Callable[[ModelSpec, BaseEngine], Awaitable[None] | None] | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "spec", "kind": "positional or keyword", "annotation": "ModelSpec", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ResidencyManager._run_hook` calls `hook`, `inspect.isawaitable`; awaits asynchronous work; returns `None`.", "implementation": "Method `ResidencyManager._run_hook` calls `hook`, `inspect.isawaitable`; awaits asynchronous work; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 396, "end_line": 407, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L396-L407", "decorators": [], "calls": [ "hook", "inspect.isawaitable" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_prepare_engine_start", "qualname": "ResidencyManager._prepare_engine_start", "full_name": "vllm_mlx.lifecycle.ResidencyManager._prepare_engine_start", "kind": "method", "signature": "async def _prepare_engine_start(self, resident: ResidentModel, engine: BaseEngine) -> None", "parameters": [ { "name": "resident", "kind": "positional or keyword", "annotation": "ResidentModel", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Run blocking startup work away from the serving event loop.", "summary": "Run blocking startup work away from the serving event loop.", "implementation": "Method `ResidencyManager._prepare_engine_start` calls `getattr`, `callable`, `uses_default_prepare`, `prepare_for_start`; awaits asynchronous work; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 409, "end_line": 445, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L409-L445", "decorators": [], "calls": [ "getattr", "callable", "uses_default_prepare", "prepare_for_start", "asyncio.create_task", "asyncio.to_thread", "asyncio.shield", "suspend_cancellation", "prepare_task.done" ], "state_reads": [ "self._lock" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_cleanup_cancelled_load", "qualname": "ResidencyManager._cleanup_cancelled_load", "full_name": "vllm_mlx.lifecycle.ResidencyManager._cleanup_cancelled_load", "kind": "method", "signature": "async def _cleanup_cancelled_load(self, resident: ResidentModel, engine: BaseEngine | None) -> None", "parameters": [ { "name": "resident", "kind": "positional or keyword", "annotation": "ResidentModel", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Stop a partially loaded engine and unwind resident state.", "summary": "Stop a partially loaded engine and unwind resident state.", "implementation": "Method `ResidencyManager._cleanup_cancelled_load` calls `suspend_cancellation`, `suppress`, `engine.stop`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 447, "end_line": 465, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L447-L465", "decorators": [], "calls": [ "suspend_cancellation", "suppress", "engine.stop" ], "state_reads": [ "self._lock" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_release_load_waiter", "qualname": "ResidencyManager._release_load_waiter", "full_name": "vllm_mlx.lifecycle.ResidencyManager._release_load_waiter", "kind": "method", "signature": "async def _release_load_waiter(self, model_key: str, task: asyncio.Task[BaseEngine]) -> None", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "task", "kind": "positional or keyword", "annotation": "asyncio.Task[BaseEngine]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Drop one waiter from a shared load, canceling abandoned solo loads.", "summary": "Drop one waiter from a shared load, canceling abandoned solo loads.", "implementation": "Method `ResidencyManager._release_load_waiter` calls `self._resident`, `task.done`, `suspend_cancellation`, `task_to_cancel.cancel`; awaits asynchronous work; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 467, "end_line": 493, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/lifecycle.py#L467-L493", "decorators": [], "calls": [ "self._resident", "task.done", "suspend_cancellation", "task_to_cancel.cancel", "suppress" ], "state_reads": [ "self._lock", "self._resident" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false } ] }, { "name": "vllm_mlx.mcp", "path": "vllm_mlx/mcp/__init__.py", "page_path": "reference/api/vllm_mlx/mcp/index.md", "docstring": "MCP (Model Context Protocol) client support for vllm-mlx.\n\nThis module provides integration with MCP servers, allowing the vllm-mlx server\nto discover and execute tools from external MCP servers.\n\nExample usage:\n from vllm_mlx.mcp import MCPClientManager, load_mcp_config\n\n config = load_mcp_config(\"./mcp.json\")\n manager = MCPClientManager(config)\n await manager.start()\n\n # Get all available tools in OpenAI format\n tools = manager.get_all_tools()\n\n # Execute a tool call\n result = await manager.execute_tool(\"filesystem__read_file\", {\"path\": \"/tmp/test.txt\"})", "summary": "MCP (Model Context Protocol) client support for vllm-mlx.", "line_count": 85, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/__init__.py#L1-L85", "members": [ "__all__" ], "symbols": [] }, { "name": "vllm_mlx.mcp.client", "path": "vllm_mlx/mcp/client.py", "page_path": "reference/api/vllm_mlx/mcp/client.md", "docstring": "MCP client for connecting to individual MCP servers.", "summary": "MCP client for connecting to individual MCP servers.", "line_count": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L1-L328", "members": [ "logger", "MCPClient" ], "symbols": [ { "name": "MCPClient", "qualname": "MCPClient", "full_name": "vllm_mlx.mcp.client.MCPClient", "kind": "class", "signature": "class MCPClient", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "MCPServerConfig", "default": "", "required": true, "description": "Server configuration" } ], "return_annotation": "MCPClient", "docstring": "Client for connecting to a single MCP server.\n\nSupports both stdio and SSE transports.", "summary": "Client for connecting to a single MCP server.", "implementation": "Class `MCPClient` declares 15 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 23, "end_line": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L23-L328", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MCPClient.__init__", "full_name": "vllm_mlx.mcp.client.MCPClient.__init__", "kind": "method", "signature": "def __init__(self, config: MCPServerConfig)", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "MCPServerConfig", "default": "", "required": true, "description": "Server configuration" } ], "return_annotation": "not annotated", "docstring": "Initialize MCP client.\n\nArgs:\n config: Server configuration", "summary": "Initialize MCP client.", "implementation": "Method `MCPClient.__init__` updates `self.config`, `self._session`, `self._read`, `self._write`; calls `asyncio.Lock`.", "documented": true, "public": false, "addressable": true, "line": 30, "end_line": 45, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L30-L45", "decorators": [], "calls": [ "asyncio.Lock" ], "state_reads": [], "state_writes": [ "self.config", "self._session", "self._read", "self._write", "self._tools", "self._state", "self._error", "self._last_connected", "self._lock" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "name", "qualname": "MCPClient.name", "full_name": "vllm_mlx.mcp.client.MCPClient.name", "kind": "method", "signature": "def name(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Get server name.", "summary": "Get server name.", "implementation": "Method `MCPClient.name` returns `self.config.name`.", "documented": true, "public": true, "addressable": true, "line": 48, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L48-L50", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.config.name", "self.config" ], "state_writes": [], "raises": [], "return_expressions": [ "self.config.name" ], "awaits": false, "yields": false }, { "name": "state", "qualname": "MCPClient.state", "full_name": "vllm_mlx.mcp.client.MCPClient.state", "kind": "method", "signature": "def state(self) -> MCPServerState", "parameters": [], "return_annotation": "MCPServerState", "docstring": "Get current connection state.", "summary": "Get current connection state.", "implementation": "Method `MCPClient.state` returns `self._state`.", "documented": true, "public": true, "addressable": true, "line": 53, "end_line": 55, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L53-L55", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._state" ], "state_writes": [], "raises": [], "return_expressions": [ "self._state" ], "awaits": false, "yields": false }, { "name": "is_connected", "qualname": "MCPClient.is_connected", "full_name": "vllm_mlx.mcp.client.MCPClient.is_connected", "kind": "method", "signature": "def is_connected(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if connected to server.", "summary": "Check if connected to server.", "implementation": "Method `MCPClient.is_connected` returns `self._state == MCPServerState.CONNECTED`.", "documented": true, "public": true, "addressable": true, "line": 58, "end_line": 60, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L58-L60", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._state" ], "state_writes": [], "raises": [], "return_expressions": [ "self._state == MCPServerState.CONNECTED" ], "awaits": false, "yields": false }, { "name": "tools", "qualname": "MCPClient.tools", "full_name": "vllm_mlx.mcp.client.MCPClient.tools", "kind": "method", "signature": "def tools(self) -> List[MCPTool]", "parameters": [], "return_annotation": "List[MCPTool]", "docstring": "Get discovered tools.", "summary": "Get discovered tools.", "implementation": "Method `MCPClient.tools` returns `self._tools`.", "documented": true, "public": true, "addressable": true, "line": 63, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L63-L65", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._tools" ], "state_writes": [], "raises": [], "return_expressions": [ "self._tools" ], "awaits": false, "yields": false }, { "name": "get_status", "qualname": "MCPClient.get_status", "full_name": "vllm_mlx.mcp.client.MCPClient.get_status", "kind": "method", "signature": "def get_status(self) -> MCPServerStatus", "parameters": [], "return_annotation": "MCPServerStatus", "docstring": "Get server status.", "summary": "Get server status.", "implementation": "Method `MCPClient.get_status` calls `MCPServerStatus`, `len`; returns `MCPServerStatus(name=self.name, state=self._state, transport=self.config.transport, tools_count=len(self._tools), error…`.", "documented": true, "public": true, "addressable": true, "line": 67, "end_line": 76, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L67-L76", "decorators": [], "calls": [ "MCPServerStatus", "len" ], "state_reads": [ "self.name", "self._state", "self.config.transport", "self.config", "self._tools", "self._error", "self._last_connected" ], "state_writes": [], "raises": [], "return_expressions": [ "MCPServerStatus(name=self.name, state=self._state, transport=self.config.transport, tools_count=len(self._tools), error…" ], "awaits": false, "yields": false }, { "name": "connect", "qualname": "MCPClient.connect", "full_name": "vllm_mlx.mcp.client.MCPClient.connect", "kind": "method", "signature": "async def connect(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Connect to the MCP server.\n\nReturns:\n True if connection successful, False otherwise", "summary": "Connect to the MCP server.", "implementation": "Method `MCPClient.connect` updates `self._state`, `self._error`, `self._last_connected`; calls `logger.info`, `self._connect_stdio`, `self._connect_sse`, `ValueError`; awaits asynchronous work; can raise `ValueError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 78, "end_line": 122, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L78-L122", "decorators": [], "calls": [ "logger.info", "self._connect_stdio", "self._connect_sse", "ValueError", "self._initialize_session", "self._discover_tools", "time.time", "len", "str", "logger.error" ], "state_reads": [ "self._lock", "self._state", "self.config.enabled", "self.config", "self.name", "self.config.transport", "self._connect_stdio", "self._connect_sse", "self._initialize_session", "self._discover_tools", "self._tools" ], "state_writes": [ "self._state", "self._error", "self._last_connected" ], "raises": [ "ValueError" ], "return_expressions": [ "True", "False" ], "awaits": true, "yields": false }, { "name": "_connect_stdio", "qualname": "MCPClient._connect_stdio", "full_name": "vllm_mlx.mcp.client.MCPClient._connect_stdio", "kind": "method", "signature": "async def _connect_stdio(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Connect via stdio transport.", "summary": "Connect via stdio transport.", "implementation": "Method `MCPClient._connect_stdio` updates `self._stdio_client`, `self._read`, `self._write`, `self._session`; calls `ImportError`, `logger.info`, `' '.join`, `StdioServerParameters`; awaits asynchronous work; can raise `ImportError`.", "documented": true, "public": false, "addressable": true, "line": 124, "end_line": 152, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L124-L152", "decorators": [], "calls": [ "ImportError", "logger.info", "' '.join", "StdioServerParameters", "stdio_client", "self._stdio_client.__aenter__", "ClientSession", "self._session.__aenter__" ], "state_reads": [ "self.name", "self.config.command", "self.config", "self.config.args", "self.config.env", "self._stdio_client.__aenter__", "self._stdio_client", "self._read", "self._write", "self._session.__aenter__", "self._session" ], "state_writes": [ "self._stdio_client", "self._read", "self._write", "self._session" ], "raises": [ "ImportError" ], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_connect_sse", "qualname": "MCPClient._connect_sse", "full_name": "vllm_mlx.mcp.client.MCPClient._connect_sse", "kind": "method", "signature": "async def _connect_sse(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Connect via SSE transport.", "summary": "Connect via SSE transport.", "implementation": "Method `MCPClient._connect_sse` updates `self._sse_client`, `self._read`, `self._write`, `self._session`; calls `ImportError`, `sse_client`, `self._sse_client.__aenter__`, `ClientSession`; awaits asynchronous work; can raise `ImportError`.", "documented": true, "public": false, "addressable": true, "line": 154, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L154-L170", "decorators": [], "calls": [ "ImportError", "sse_client", "self._sse_client.__aenter__", "ClientSession", "self._session.__aenter__" ], "state_reads": [ "self.config.url", "self.config", "self._sse_client.__aenter__", "self._sse_client", "self._read", "self._write", "self._session.__aenter__", "self._session" ], "state_writes": [ "self._sse_client", "self._read", "self._write", "self._session" ], "raises": [ "ImportError" ], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_initialize_session", "qualname": "MCPClient._initialize_session", "full_name": "vllm_mlx.mcp.client.MCPClient._initialize_session", "kind": "method", "signature": "async def _initialize_session(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Initialize the MCP session.", "summary": "Initialize the MCP session.", "implementation": "Method `MCPClient._initialize_session` calls `RuntimeError`, `self._session.initialize`, `logger.debug`; awaits asynchronous work; can raise `RuntimeError`.", "documented": true, "public": false, "addressable": true, "line": 172, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L172-L183", "decorators": [], "calls": [ "RuntimeError", "self._session.initialize", "logger.debug" ], "state_reads": [ "self._session", "self._session.initialize", "self.name" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_discover_tools", "qualname": "MCPClient._discover_tools", "full_name": "vllm_mlx.mcp.client.MCPClient._discover_tools", "kind": "method", "signature": "async def _discover_tools(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Discover available tools from the server.", "summary": "Discover available tools from the server.", "implementation": "Method `MCPClient._discover_tools` updates `self._tools`; calls `RuntimeError`, `self._session.list_tools`, `MCPTool`, `hasattr`; awaits asynchronous work; can raise `RuntimeError`.", "documented": true, "public": false, "addressable": true, "line": 185, "end_line": 208, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L185-L208", "decorators": [], "calls": [ "RuntimeError", "self._session.list_tools", "MCPTool", "hasattr", "self._tools.append", "logger.debug", "logger.warning" ], "state_reads": [ "self._session", "self._session.list_tools", "self.name", "self._tools.append", "self._tools" ], "state_writes": [ "self._tools" ], "raises": [ "RuntimeError" ], "return_expressions": [], "awaits": true, "yields": false }, { "name": "disconnect", "qualname": "MCPClient.disconnect", "full_name": "vllm_mlx.mcp.client.MCPClient.disconnect", "kind": "method", "signature": "async def disconnect(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Disconnect from the MCP server.", "summary": "Disconnect from the MCP server.", "implementation": "Method `MCPClient.disconnect` updates `self._session`, `self._stdio_client`, `self._sse_client`, `self._state`; calls `self._session.__aexit__`, `hasattr`, `self._stdio_client.__aexit__`, `self._sse_client.__aexit__`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 210, "end_line": 235, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L210-L235", "decorators": [], "calls": [ "self._session.__aexit__", "hasattr", "self._stdio_client.__aexit__", "self._sse_client.__aexit__", "logger.warning", "logger.info" ], "state_reads": [ "self._lock", "self._state", "self._session", "self._session.__aexit__", "self._stdio_client", "self._stdio_client.__aexit__", "self._sse_client", "self._sse_client.__aexit__", "self.name" ], "state_writes": [ "self._session", "self._stdio_client", "self._sse_client", "self._state", "self._tools" ], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "call_tool", "qualname": "MCPClient.call_tool", "full_name": "vllm_mlx.mcp.client.MCPClient.call_tool", "kind": "method", "signature": "async def call_tool(self, tool_name: str, arguments: Dict[str, Any], timeout: Optional[float]=None) -> MCPToolResult", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the tool (without server prefix)" }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Tool arguments" }, { "name": "timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional timeout in seconds" } ], "return_annotation": "MCPToolResult", "docstring": "Call a tool on the MCP server.\n\nArgs:\n tool_name: Name of the tool (without server prefix)\n arguments: Tool arguments\n timeout: Optional timeout in seconds\n\nReturns:\n MCPToolResult with the result or error", "summary": "Call a tool on the MCP server.", "implementation": "Method `MCPClient.call_tool` calls `MCPToolResult`, `asyncio.wait_for`, `self._session.call_tool`, `self._extract_content`; awaits asynchronous work; has 5 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 237, "end_line": 301, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L237-L301", "decorators": [], "calls": [ "MCPToolResult", "asyncio.wait_for", "self._session.call_tool", "self._extract_content", "hasattr", "str" ], "state_reads": [ "self.is_connected", "self.name", "self._session", "self.config.timeout", "self.config", "self._session.call_tool", "self._extract_content" ], "state_writes": [], "raises": [], "return_expressions": [ "MCPToolResult(tool_name=tool_name, content=None, is_error=True, error_message=f\"Not connected to server '{self.name}'\")", "MCPToolResult(tool_name=tool_name, content=None, is_error=True, error_message='Session not initialized')", "MCPToolResult(tool_name=tool_name, content=content, is_error=result.isError if hasattr(result, 'isError') else False)", "MCPToolResult(tool_name=tool_name, content=None, is_error=True, error_message=f'Tool call timed out after {timeout}s')", "MCPToolResult(tool_name=tool_name, content=None, is_error=True, error_message=str(e))" ], "awaits": true, "yields": false }, { "name": "_extract_content", "qualname": "MCPClient._extract_content", "full_name": "vllm_mlx.mcp.client.MCPClient._extract_content", "kind": "method", "signature": "def _extract_content(self, result) -> Any", "parameters": [ { "name": "result", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Extract content from MCP tool result.", "summary": "Extract content from MCP tool result.", "implementation": "Method `MCPClient._extract_content` calls `hasattr`, `contents.append`, `str`, `len`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 303, "end_line": 321, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L303-L321", "decorators": [], "calls": [ "hasattr", "contents.append", "str", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "contents[0]", "contents" ], "awaits": false, "yields": false }, { "name": "refresh_tools", "qualname": "MCPClient.refresh_tools", "full_name": "vllm_mlx.mcp.client.MCPClient.refresh_tools", "kind": "method", "signature": "async def refresh_tools(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Refresh the list of available tools.", "summary": "Refresh the list of available tools.", "implementation": "Method `MCPClient.refresh_tools` calls `self._discover_tools`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 323, "end_line": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/client.py#L323-L328", "decorators": [], "calls": [ "self._discover_tools" ], "state_reads": [ "self.is_connected", "self._discover_tools" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false } ] }, { "name": "vllm_mlx.mcp.config", "path": "vllm_mlx/mcp/config.py", "page_path": "reference/api/vllm_mlx/mcp/config.md", "docstring": "MCP configuration loading and validation.", "summary": "MCP configuration loading and validation.", "line_count": 199, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/config.py#L1-L199", "members": [ "logger", "CONFIG_SEARCH_PATHS", "CONFIG_ENV_VAR", "load_mcp_config", "_find_config_file", "validate_config", "create_example_config" ], "symbols": [ { "name": "load_mcp_config", "qualname": "load_mcp_config", "full_name": "vllm_mlx.mcp.config.load_mcp_config", "kind": "function", "signature": "def load_mcp_config(path: Optional[Union[str, Path]]=None) -> MCPConfig", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Optional[Union[str, Path]]", "default": "None", "required": false, "description": "Optional explicit path to config file" } ], "return_annotation": "MCPConfig", "docstring": "Load MCP configuration from file.\n\nSearch order:\n1. Explicit path argument\n2. VLLM_MLX_MCP_CONFIG environment variable\n3. ~/.config/vllm-mlx/mcp.json or mcp.yaml\n\nArgs:\n path: Optional explicit path to config file\n\nReturns:\n MCPConfig object\n\nRaises:\n FileNotFoundError: If no config file found\n ValueError: If config is invalid", "summary": "Load MCP configuration from file.", "implementation": "Function `load_mcp_config` calls `_find_config_file`, `logger.info`, `MCPConfig`, `Path(config_path).expanduser`; can raise `ImportError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 26, "end_line": 70, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/config.py#L26-L70", "decorators": [], "calls": [ "_find_config_file", "logger.info", "MCPConfig", "Path(config_path).expanduser", "Path", "config_path.read_text", "yaml.safe_load", "ImportError", "json.loads", "validate_config" ], "state_reads": [], "state_writes": [], "raises": [ "ImportError" ], "return_expressions": [ "MCPConfig()", "validate_config(data)" ], "awaits": false, "yields": false }, { "name": "_find_config_file", "qualname": "_find_config_file", "full_name": "vllm_mlx.mcp.config._find_config_file", "kind": "function", "signature": "def _find_config_file(explicit_path: Optional[Union[str, Path]]=None) -> Optional[Path]", "parameters": [ { "name": "explicit_path", "kind": "positional or keyword", "annotation": "Optional[Union[str, Path]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "Optional[Path]", "docstring": "Find the config file to use.", "summary": "Find the config file to use.", "implementation": "Function `_find_config_file` calls `Path(explicit_path).expanduser`, `Path`, `path.exists`, `FileNotFoundError`; can raise `FileNotFoundError`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 73, "end_line": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/config.py#L73-L98", "decorators": [], "calls": [ "Path(explicit_path).expanduser", "Path", "path.exists", "FileNotFoundError", "os.environ.get", "Path(env_path).expanduser", "logger.warning", "Path(search_path).expanduser" ], "state_reads": [], "state_writes": [], "raises": [ "FileNotFoundError" ], "return_expressions": [ "path", "None" ], "awaits": false, "yields": false }, { "name": "validate_config", "qualname": "validate_config", "full_name": "vllm_mlx.mcp.config.validate_config", "kind": "function", "signature": "def validate_config(data: Dict[str, Any]) -> MCPConfig", "parameters": [ { "name": "data", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Raw configuration dictionary" } ], "return_annotation": "MCPConfig", "docstring": "Validate and parse configuration dictionary.\n\nArgs:\n data: Raw configuration dictionary\n\nReturns:\n Validated MCPConfig object\n\nRaises:\n ValueError: If configuration is invalid", "summary": "Validate and parse configuration dictionary.", "implementation": "Function `validate_config` calls `isinstance`, `ValueError`, `data.get`, `servers_data.items`; can raise `ValueError`; returns `MCPConfig(servers=servers, max_tool_calls=max_tool_calls, default_timeout=default_timeout, allowed_high_risk_tools=set(…`.", "documented": true, "public": true, "addressable": true, "line": 101, "end_line": 163, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/config.py#L101-L163", "decorators": [], "calls": [ "isinstance", "ValueError", "data.get", "servers_data.items", "server_data.copy", "MCPServerConfig", "any", "tool.strip", "MCPConfig", "set" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "MCPConfig(servers=servers, max_tool_calls=max_tool_calls, default_timeout=default_timeout, allowed_high_risk_tools=set(…" ], "awaits": false, "yields": false }, { "name": "create_example_config", "qualname": "create_example_config", "full_name": "vllm_mlx.mcp.config.create_example_config", "kind": "function", "signature": "def create_example_config() -> str", "parameters": [], "return_annotation": "str", "docstring": "Create an example MCP configuration.\n\nReturns:\n JSON string with example configuration", "summary": "Create an example MCP configuration.", "implementation": "Function `create_example_config` calls `json.dumps`; returns `json.dumps(example, indent=2)`.", "documented": true, "public": true, "addressable": true, "line": 166, "end_line": 199, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/config.py#L166-L199", "decorators": [], "calls": [ "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.dumps(example, indent=2)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.mcp.executor", "path": "vllm_mlx/mcp/executor.py", "page_path": "reference/api/vllm_mlx/mcp/executor.md", "docstring": "Tool executor for handling tool calls from model responses.", "summary": "Tool executor for handling tool calls from model responses.", "line_count": 500, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L1-L500", "members": [ "logger", "ToolArgumentValidationError", "validate_tool_arguments", "ToolExecutor", "execute_single_tool" ], "symbols": [ { "name": "ToolArgumentValidationError", "qualname": "ToolArgumentValidationError", "full_name": "vllm_mlx.mcp.executor.ToolArgumentValidationError", "kind": "class", "signature": "class ToolArgumentValidationError(Exception)", "parameters": [], "return_annotation": "ToolArgumentValidationError", "docstring": "Raised when tool arguments fail validation against schema.", "summary": "Raised when tool arguments fail validation against schema.", "implementation": "Class `ToolArgumentValidationError` derives from `Exception` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 25, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L22-L25", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "validate_tool_arguments", "qualname": "validate_tool_arguments", "full_name": "vllm_mlx.mcp.executor.validate_tool_arguments", "kind": "function", "signature": "def validate_tool_arguments(tool: MCPTool, arguments: Dict[str, Any], strict: bool=True) -> None", "parameters": [ { "name": "tool", "kind": "positional or keyword", "annotation": "MCPTool", "default": "", "required": true, "description": "The MCP tool with input_schema" }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Arguments to validate" }, { "name": "strict", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, raise exception on validation failure" } ], "return_annotation": "None", "docstring": "Validate tool arguments against the tool's input schema.\n\nArgs:\n tool: The MCP tool with input_schema\n arguments: Arguments to validate\n strict: If True, raise exception on validation failure\n\nRaises:\n ToolArgumentValidationError: If validation fails and strict=True", "summary": "Validate tool arguments against the tool's input schema.", "implementation": "Function `validate_tool_arguments` calls `logger.debug`, `jsonschema.validate`, `'.'.join`, `str`; can raise `ToolArgumentValidationError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 61, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L28-L61", "decorators": [], "calls": [ "logger.debug", "jsonschema.validate", "'.'.join", "str", "logger.warning", "ToolArgumentValidationError" ], "state_reads": [], "state_writes": [], "raises": [ "ToolArgumentValidationError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "ToolExecutor", "qualname": "ToolExecutor", "full_name": "vllm_mlx.mcp.executor.ToolExecutor", "kind": "class", "signature": "class ToolExecutor", "parameters": [ { "name": "manager", "kind": "positional or keyword", "annotation": "MCPClientManager", "default": "", "required": true, "description": "MCP client manager" }, { "name": "max_parallel", "kind": "positional or keyword", "annotation": "int", "default": "5", "required": false, "description": "Maximum parallel tool executions" }, { "name": "default_timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Default timeout for tool calls" }, { "name": "validate_arguments", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, validate arguments against tool schemas" }, { "name": "sandbox", "kind": "positional or keyword", "annotation": "Optional[ToolSandbox]", "default": "None", "required": false, "description": "Optional tool sandbox for security controls. Uses global if None." } ], "return_annotation": "ToolExecutor", "docstring": "Handles execution of tool calls from model responses.\n\nProvides utilities for:\n- Extracting tool calls from responses\n- Executing multiple tool calls (parallel or sequential)\n- Formatting results for conversation\n- Validating tool arguments against schemas", "summary": "Handles execution of tool calls from model responses.", "implementation": "Class `ToolExecutor` declares 11 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 64, "end_line": 479, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L64-L479", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ToolExecutor.__init__", "full_name": "vllm_mlx.mcp.executor.ToolExecutor.__init__", "kind": "method", "signature": "def __init__(self, manager: MCPClientManager, max_parallel: int=5, default_timeout: Optional[float]=None, validate_arguments: bool=True, sandbox: Optional[ToolSandbox]=None)", "parameters": [ { "name": "manager", "kind": "positional or keyword", "annotation": "MCPClientManager", "default": "", "required": true, "description": "MCP client manager" }, { "name": "max_parallel", "kind": "positional or keyword", "annotation": "int", "default": "5", "required": false, "description": "Maximum parallel tool executions" }, { "name": "default_timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Default timeout for tool calls" }, { "name": "validate_arguments", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, validate arguments against tool schemas" }, { "name": "sandbox", "kind": "positional or keyword", "annotation": "Optional[ToolSandbox]", "default": "None", "required": false, "description": "Optional tool sandbox for security controls. Uses global if None." } ], "return_annotation": "not annotated", "docstring": "Initialize tool executor.\n\nArgs:\n manager: MCP client manager\n max_parallel: Maximum parallel tool executions\n default_timeout: Default timeout for tool calls\n validate_arguments: If True, validate arguments against tool schemas\n sandbox: Optional tool sandbox for security controls. Uses global if None.", "summary": "Initialize tool executor.", "implementation": "Method `ToolExecutor.__init__` updates `self.manager`, `self.max_parallel`, `self.default_timeout`, `self.validate_arguments`; calls `get_sandbox`.", "documented": true, "public": false, "addressable": true, "line": 75, "end_line": 97, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L75-L97", "decorators": [], "calls": [ "get_sandbox" ], "state_reads": [], "state_writes": [ "self.manager", "self.max_parallel", "self.default_timeout", "self.validate_arguments", "self.sandbox" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "execute_tool_calls", "qualname": "ToolExecutor.execute_tool_calls", "full_name": "vllm_mlx.mcp.executor.ToolExecutor.execute_tool_calls", "kind": "method", "signature": "async def execute_tool_calls(self, tool_calls: List[Dict[str, Any]], parallel: bool=True) -> List[Tuple[MCPToolResult, str]]", "parameters": [ { "name": "tool_calls", "kind": "positional or keyword", "annotation": "List[Dict[str, Any]]", "default": "", "required": true, "description": "List of OpenAI tool call objects" }, { "name": "parallel", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Execute in parallel (True) or sequential (False)" } ], "return_annotation": "List[Tuple[MCPToolResult, str]]", "docstring": "Execute multiple tool calls.\n\nArgs:\n tool_calls: List of OpenAI tool call objects\n parallel: Execute in parallel (True) or sequential (False)\n\nReturns:\n List of (MCPToolResult, tool_call_id) tuples", "summary": "Execute multiple tool calls.", "implementation": "Method `ToolExecutor.execute_tool_calls` calls `self._execute_parallel`, `self._execute_sequential`; awaits asynchronous work; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 99, "end_line": 120, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L99-L120", "decorators": [], "calls": [ "self._execute_parallel", "self._execute_sequential" ], "state_reads": [ "self._execute_parallel", "self._execute_sequential" ], "state_writes": [], "raises": [], "return_expressions": [ "[]", "await self._execute_parallel(tool_calls)", "await self._execute_sequential(tool_calls)" ], "awaits": true, "yields": false }, { "name": "_get_tool_by_name", "qualname": "ToolExecutor._get_tool_by_name", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._get_tool_by_name", "kind": "method", "signature": "def _get_tool_by_name(self, full_name: str) -> Optional[MCPTool]", "parameters": [ { "name": "full_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[MCPTool]", "docstring": "Get a tool by its full name (server__tool or just tool).", "summary": "Get a tool by its full name (server__tool or just tool).", "implementation": "Method `ToolExecutor._get_tool_by_name` calls `self.manager.get_all_tools`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 122, "end_line": 132, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L122-L132", "decorators": [], "calls": [ "self.manager.get_all_tools" ], "state_reads": [ "self.manager.get_all_tools", "self.manager" ], "state_writes": [], "raises": [], "return_expressions": [ "tool", "None" ], "awaits": false, "yields": false }, { "name": "_validate_tool_call", "qualname": "ToolExecutor._validate_tool_call", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._validate_tool_call", "kind": "method", "signature": "def _validate_tool_call(self, tool_call: Dict[str, Any]) -> Optional[str]", "parameters": [ { "name": "tool_call", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[str]", "docstring": "Validate a tool call's arguments against the tool's schema.\n\nReturns:\n Error message if validation fails, None if valid", "summary": "Validate a tool call's arguments against the tool's schema.", "implementation": "Method `ToolExecutor._validate_tool_call` calls `tool_call.get`, `func.get`, `isinstance`, `json.loads`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 134, "end_line": 165, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L134-L165", "decorators": [], "calls": [ "tool_call.get", "func.get", "isinstance", "json.loads", "self._get_tool_by_name", "validate_tool_arguments", "str" ], "state_reads": [ "self.validate_arguments", "self._get_tool_by_name" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "f\"Invalid JSON in arguments for tool '{name}'\"", "str(e)" ], "awaits": false, "yields": false }, { "name": "_validate_sandbox", "qualname": "ToolExecutor._validate_sandbox", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._validate_sandbox", "kind": "method", "signature": "def _validate_sandbox(self, tool_name: str, server_name: str, arguments: Dict[str, Any]) -> Optional[str]", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[str]", "docstring": "Validate tool execution against sandbox policy.\n\nReturns:\n Error message if blocked, None if allowed", "summary": "Validate tool execution against sandbox policy.", "implementation": "Method `ToolExecutor._validate_sandbox` calls `self.sandbox.validate_tool_execution`, `str`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 167, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L167-L183", "decorators": [], "calls": [ "self.sandbox.validate_tool_execution", "str" ], "state_reads": [ "self.sandbox.validate_tool_execution", "self.sandbox" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "str(e)" ], "awaits": false, "yields": false }, { "name": "_get_server_for_tool", "qualname": "ToolExecutor._get_server_for_tool", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._get_server_for_tool", "kind": "method", "signature": "def _get_server_for_tool(self, full_name: str) -> str", "parameters": [ { "name": "full_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Extract server name from full tool name or find it.", "summary": "Extract server name from full tool name or find it.", "implementation": "Method `ToolExecutor._get_server_for_tool` calls `full_name.split`, `self.manager.get_all_tools`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 185, "end_line": 193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L185-L193", "decorators": [], "calls": [ "full_name.split", "self.manager.get_all_tools" ], "state_reads": [ "self.manager.get_all_tools", "self.manager" ], "state_writes": [], "raises": [], "return_expressions": [ "full_name.split('__')[0]", "tool.server_name", "'unknown'" ], "awaits": false, "yields": false }, { "name": "_execute_parallel", "qualname": "ToolExecutor._execute_parallel", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._execute_parallel", "kind": "method", "signature": "async def _execute_parallel(self, tool_calls: List[Dict[str, Any]]) -> List[Tuple[MCPToolResult, str]]", "parameters": [ { "name": "tool_calls", "kind": "positional or keyword", "annotation": "List[Dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "List[Tuple[MCPToolResult, str]]", "docstring": "Execute tool calls in parallel with concurrency limit.", "summary": "Execute tool calls in parallel with concurrency limit.", "implementation": "Method `ToolExecutor._execute_parallel` calls `asyncio.Semaphore`, `execute_with_semaphore`, `asyncio.gather`, `enumerate`; awaits asynchronous work; returns `processed`.", "documented": true, "public": false, "addressable": true, "line": 195, "end_line": 305, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L195-L305", "decorators": [], "calls": [ "asyncio.Semaphore", "execute_with_semaphore", "asyncio.gather", "enumerate", "tool_calls[i].get", "isinstance", "processed.append", "MCPToolResult", "tool_calls[i].get('function', {}).get", "str" ], "state_reads": [ "self.max_parallel" ], "state_writes": [], "raises": [], "return_expressions": [ "processed" ], "awaits": true, "yields": false }, { "name": "execute_with_semaphore", "qualname": "ToolExecutor._execute_parallel.execute_with_semaphore", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._execute_parallel.execute_with_semaphore", "kind": "nested function", "signature": "async def execute_with_semaphore(tool_call: Dict[str, Any])", "parameters": [ { "name": "tool_call", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `ToolExecutor._execute_parallel.execute_with_semaphore` calls `tool_call.get`, `func.get`, `isinstance`, `json.loads`; awaits asynchronous work; has 3 explicit return paths.", "implementation": "Nested Function `ToolExecutor._execute_parallel.execute_with_semaphore` calls `tool_call.get`, `func.get`, `isinstance`, `json.loads`; awaits asynchronous work; has 3 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 202, "end_line": 281, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L202-L281", "decorators": [], "calls": [ "tool_call.get", "func.get", "isinstance", "json.loads", "self._get_server_for_tool", "name.split", "self._validate_tool_call", "self.sandbox.record_execution", "MCPToolResult", "self._validate_sandbox", "time.time", "self.manager.execute_tool_call" ], "state_reads": [ "self._get_server_for_tool", "self._validate_tool_call", "self.sandbox.record_execution", "self.sandbox", "self._validate_sandbox", "self.manager.execute_tool_call", "self.manager", "self.default_timeout" ], "state_writes": [], "raises": [], "return_expressions": [ "(MCPToolResult(tool_name=name, content=None, is_error=True, error_message=validation_error), call_id)", "(MCPToolResult(tool_name=name, content=None, is_error=True, error_message=sandbox_error), call_id)", "(result, call_id)" ], "awaits": true, "yields": false }, { "name": "_execute_sequential", "qualname": "ToolExecutor._execute_sequential", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._execute_sequential", "kind": "method", "signature": "async def _execute_sequential(self, tool_calls: List[Dict[str, Any]]) -> List[Tuple[MCPToolResult, str]]", "parameters": [ { "name": "tool_calls", "kind": "positional or keyword", "annotation": "List[Dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "List[Tuple[MCPToolResult, str]]", "docstring": "Execute tool calls sequentially.", "summary": "Execute tool calls sequentially.", "implementation": "Method `ToolExecutor._execute_sequential` calls `tool_call.get`, `func.get`, `isinstance`, `json.loads`; awaits asynchronous work; returns `results`.", "documented": true, "public": false, "addressable": true, "line": 307, "end_line": 415, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L307-L415", "decorators": [], "calls": [ "tool_call.get", "func.get", "isinstance", "json.loads", "self._get_server_for_tool", "name.split", "self._validate_tool_call", "self.sandbox.record_execution", "results.append", "MCPToolResult", "self._validate_sandbox", "time.time", "self.manager.execute_tool_call", "str" ], "state_reads": [ "self._get_server_for_tool", "self._validate_tool_call", "self.sandbox.record_execution", "self.sandbox", "self._validate_sandbox", "self.manager.execute_tool_call", "self.manager", "self.default_timeout" ], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": true, "yields": false }, { "name": "execute_and_format", "qualname": "ToolExecutor.execute_and_format", "full_name": "vllm_mlx.mcp.executor.ToolExecutor.execute_and_format", "kind": "method", "signature": "async def execute_and_format(self, tool_calls: List[Dict[str, Any]], parallel: bool=True) -> List[Dict[str, Any]]", "parameters": [ { "name": "tool_calls", "kind": "positional or keyword", "annotation": "List[Dict[str, Any]]", "default": "", "required": true, "description": "List of OpenAI tool call objects" }, { "name": "parallel", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Execute in parallel" } ], "return_annotation": "List[Dict[str, Any]]", "docstring": "Execute tool calls and format results as messages.\n\nArgs:\n tool_calls: List of OpenAI tool call objects\n parallel: Execute in parallel\n\nReturns:\n List of tool result messages ready for conversation", "summary": "Execute tool calls and format results as messages.", "implementation": "Method `ToolExecutor.execute_and_format` calls `self.execute_tool_calls`, `format_tool_result`; awaits asynchronous work; returns `[format_tool_result(result, call_id) for result, call_id in results]`.", "documented": true, "public": true, "addressable": true, "line": 417, "end_line": 433, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L417-L433", "decorators": [], "calls": [ "self.execute_tool_calls", "format_tool_result" ], "state_reads": [ "self.execute_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "[format_tool_result(result, call_id) for result, call_id in results]" ], "awaits": true, "yields": false }, { "name": "extract_and_validate", "qualname": "ToolExecutor.extract_and_validate", "full_name": "vllm_mlx.mcp.executor.ToolExecutor.extract_and_validate", "kind": "method", "signature": "def extract_and_validate(self, response: Dict[str, Any]) -> Tuple[List[Dict[str, Any]], bool]", "parameters": [ { "name": "response", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Model response in OpenAI format" } ], "return_annotation": "Tuple[List[Dict[str, Any]], bool]", "docstring": "Extract tool calls from response and validate them.\n\nArgs:\n response: Model response in OpenAI format\n\nReturns:\n Tuple of (tool_calls, all_valid)", "summary": "Extract tool calls from response and validate them.", "implementation": "Method `ToolExecutor.extract_and_validate` calls `extract_tool_calls`, `tc.get`, `func.get`, `self._tool_exists`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 435, "end_line": 464, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L435-L464", "decorators": [], "calls": [ "extract_tool_calls", "tc.get", "func.get", "self._tool_exists", "logger.warning" ], "state_reads": [ "self._tool_exists" ], "state_writes": [], "raises": [], "return_expressions": [ "([], True)", "(tool_calls, all_valid)" ], "awaits": false, "yields": false }, { "name": "_tool_exists", "qualname": "ToolExecutor._tool_exists", "full_name": "vllm_mlx.mcp.executor.ToolExecutor._tool_exists", "kind": "method", "signature": "def _tool_exists(self, full_name: str) -> bool", "parameters": [ { "name": "full_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if a tool exists in any connected server.", "summary": "Check if a tool exists in any connected server.", "implementation": "Method `ToolExecutor._tool_exists` calls `self.manager.get_all_tools`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 466, "end_line": 479, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L466-L479", "decorators": [], "calls": [ "self.manager.get_all_tools" ], "state_reads": [ "self.manager.get_all_tools", "self.manager" ], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "execute_single_tool", "qualname": "execute_single_tool", "full_name": "vllm_mlx.mcp.executor.execute_single_tool", "kind": "function", "signature": "async def execute_single_tool(manager: MCPClientManager, tool_name: str, arguments: Dict[str, Any], timeout: Optional[float]=None) -> MCPToolResult", "parameters": [ { "name": "manager", "kind": "positional or keyword", "annotation": "MCPClientManager", "default": "", "required": true, "description": "MCP client manager" }, { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Full tool name (server__tool)" }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Tool arguments" }, { "name": "timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional timeout" } ], "return_annotation": "MCPToolResult", "docstring": "Convenience function to execute a single tool.\n\nArgs:\n manager: MCP client manager\n tool_name: Full tool name (server__tool)\n arguments: Tool arguments\n timeout: Optional timeout\n\nReturns:\n MCPToolResult", "summary": "Convenience function to execute a single tool.", "implementation": "Function `execute_single_tool` calls `manager.execute_tool`; awaits asynchronous work; returns `await manager.execute_tool(tool_name, arguments, timeout)`.", "documented": true, "public": true, "addressable": true, "line": 482, "end_line": 500, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/executor.py#L482-L500", "decorators": [], "calls": [ "manager.execute_tool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "await manager.execute_tool(tool_name, arguments, timeout)" ], "awaits": true, "yields": false } ] }, { "name": "vllm_mlx.mcp.manager", "path": "vllm_mlx/mcp/manager.py", "page_path": "reference/api/vllm_mlx/mcp/manager.md", "docstring": "MCP Client Manager for handling multiple MCP server connections.", "summary": "MCP Client Manager for handling multiple MCP server connections.", "line_count": 301, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L1-L301", "members": [ "logger", "MCPClientManager" ], "symbols": [ { "name": "MCPClientManager", "qualname": "MCPClientManager", "full_name": "vllm_mlx.mcp.manager.MCPClientManager", "kind": "class", "signature": "class MCPClientManager", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "MCPConfig", "default": "", "required": true, "description": "MCP configuration with server definitions" } ], "return_annotation": "MCPClientManager", "docstring": "Manages multiple MCP server connections.\n\nProvides a unified interface for:\n- Connecting to multiple MCP servers\n- Discovering and aggregating tools\n- Executing tool calls\n- Managing connection lifecycle", "summary": "Manages multiple MCP server connections.", "implementation": "Class `MCPClientManager` declares 14 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 301, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L22-L301", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MCPClientManager.__init__", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.__init__", "kind": "method", "signature": "def __init__(self, config: MCPConfig)", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "MCPConfig", "default": "", "required": true, "description": "MCP configuration with server definitions" } ], "return_annotation": "not annotated", "docstring": "Initialize MCP Client Manager.\n\nArgs:\n config: MCP configuration with server definitions", "summary": "Initialize MCP Client Manager.", "implementation": "Method `MCPClientManager.__init__` updates `self.config`, `self._clients`, `self._started`, `self._lock`; calls `asyncio.Lock`, `config.servers.items`, `MCPClient`.", "documented": true, "public": false, "addressable": true, "line": 33, "end_line": 47, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L33-L47", "decorators": [], "calls": [ "asyncio.Lock", "config.servers.items", "MCPClient" ], "state_reads": [ "self._clients" ], "state_writes": [ "self.config", "self._clients", "self._started", "self._lock" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_started", "qualname": "MCPClientManager.is_started", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.is_started", "kind": "method", "signature": "def is_started(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if manager has been started.", "summary": "Check if manager has been started.", "implementation": "Method `MCPClientManager.is_started` returns `self._started`.", "documented": true, "public": true, "addressable": true, "line": 50, "end_line": 52, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L50-L52", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._started" ], "state_writes": [], "raises": [], "return_expressions": [ "self._started" ], "awaits": false, "yields": false }, { "name": "start", "qualname": "MCPClientManager.start", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.start", "kind": "method", "signature": "async def start(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Start the manager and connect to all enabled servers.\n\nConnections are made in parallel for faster startup.", "summary": "Start the manager and connect to all enabled servers.", "implementation": "Method `MCPClientManager.start` updates `self._started`; calls `logger.info`, `len`, `client.connect`, `self._clients.values`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 54, "end_line": 96, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L54-L96", "decorators": [], "calls": [ "logger.info", "len", "client.connect", "self._clients.values", "asyncio.gather", "zip", "isinstance", "logger.error", "sum" ], "state_reads": [ "self._lock", "self._started", "self._clients", "self._clients.values" ], "state_writes": [ "self._started" ], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "stop", "qualname": "MCPClientManager.stop", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.stop", "kind": "method", "signature": "async def stop(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Stop the manager and disconnect from all servers.", "summary": "Stop the manager and disconnect from all servers.", "implementation": "Method `MCPClientManager.stop` updates `self._started`; calls `logger.info`, `client.disconnect`, `self._clients.values`, `asyncio.gather`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 98, "end_line": 112, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L98-L112", "decorators": [], "calls": [ "logger.info", "client.disconnect", "self._clients.values", "asyncio.gather" ], "state_reads": [ "self._lock", "self._started", "self._clients.values", "self._clients" ], "state_writes": [ "self._started" ], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "get_all_tools", "qualname": "MCPClientManager.get_all_tools", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.get_all_tools", "kind": "method", "signature": "def get_all_tools(self) -> List[MCPTool]", "parameters": [], "return_annotation": "List[MCPTool]", "docstring": "Get all tools from all connected servers.\n\nReturns:\n List of MCPTool instances", "summary": "Get all tools from all connected servers.", "implementation": "Method `MCPClientManager.get_all_tools` calls `self._clients.values`, `tools.extend`; returns `tools`.", "documented": true, "public": true, "addressable": true, "line": 114, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L114-L125", "decorators": [], "calls": [ "self._clients.values", "tools.extend" ], "state_reads": [ "self._clients.values", "self._clients" ], "state_writes": [], "raises": [], "return_expressions": [ "tools" ], "awaits": false, "yields": false }, { "name": "get_all_tools_openai", "qualname": "MCPClientManager.get_all_tools_openai", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.get_all_tools_openai", "kind": "method", "signature": "def get_all_tools_openai(self) -> List[Dict[str, Any]]", "parameters": [], "return_annotation": "List[Dict[str, Any]]", "docstring": "Get all tools in OpenAI function calling format.\n\nReturns:\n List of OpenAI-compatible tool definitions", "summary": "Get all tools in OpenAI function calling format.", "implementation": "Method `MCPClientManager.get_all_tools_openai` calls `mcp_tools_to_openai`, `self.get_all_tools`; returns `mcp_tools_to_openai(self.get_all_tools())`.", "documented": true, "public": true, "addressable": true, "line": 127, "end_line": 134, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L127-L134", "decorators": [], "calls": [ "mcp_tools_to_openai", "self.get_all_tools" ], "state_reads": [ "self.get_all_tools" ], "state_writes": [], "raises": [], "return_expressions": [ "mcp_tools_to_openai(self.get_all_tools())" ], "awaits": false, "yields": false }, { "name": "get_merged_tools", "qualname": "MCPClientManager.get_merged_tools", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.get_merged_tools", "kind": "method", "signature": "def get_merged_tools(self, user_tools: Optional[List[Dict[str, Any]]]=None) -> List[Dict[str, Any]]", "parameters": [ { "name": "user_tools", "kind": "positional or keyword", "annotation": "Optional[List[Dict[str, Any]]]", "default": "None", "required": false, "description": "Optional user-provided tools in OpenAI format" } ], "return_annotation": "List[Dict[str, Any]]", "docstring": "Get MCP tools merged with user-provided tools.\n\nUser tools take precedence on name conflicts.\n\nArgs:\n user_tools: Optional user-provided tools in OpenAI format\n\nReturns:\n Combined list of tools in OpenAI format", "summary": "Get MCP tools merged with user-provided tools.", "implementation": "Method `MCPClientManager.get_merged_tools` calls `merge_tools`, `self.get_all_tools`; returns `merge_tools(self.get_all_tools(), user_tools)`.", "documented": true, "public": true, "addressable": true, "line": 136, "end_line": 151, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L136-L151", "decorators": [], "calls": [ "merge_tools", "self.get_all_tools" ], "state_reads": [ "self.get_all_tools" ], "state_writes": [], "raises": [], "return_expressions": [ "merge_tools(self.get_all_tools(), user_tools)" ], "awaits": false, "yields": false }, { "name": "get_server_status", "qualname": "MCPClientManager.get_server_status", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.get_server_status", "kind": "method", "signature": "def get_server_status(self) -> List[MCPServerStatus]", "parameters": [], "return_annotation": "List[MCPServerStatus]", "docstring": "Get status of all servers.\n\nReturns:\n List of MCPServerStatus for each server", "summary": "Get status of all servers.", "implementation": "Method `MCPClientManager.get_server_status` calls `client.get_status`, `self._clients.values`; returns `[client.get_status() for client in self._clients.values()]`.", "documented": true, "public": true, "addressable": true, "line": 153, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L153-L160", "decorators": [], "calls": [ "client.get_status", "self._clients.values" ], "state_reads": [ "self._clients.values", "self._clients" ], "state_writes": [], "raises": [], "return_expressions": [ "[client.get_status() for client in self._clients.values()]" ], "awaits": false, "yields": false }, { "name": "get_client", "qualname": "MCPClientManager.get_client", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.get_client", "kind": "method", "signature": "def get_client(self, server_name: str) -> Optional[MCPClient]", "parameters": [ { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the server" } ], "return_annotation": "Optional[MCPClient]", "docstring": "Get client for a specific server.\n\nArgs:\n server_name: Name of the server\n\nReturns:\n MCPClient instance or None if not found", "summary": "Get client for a specific server.", "implementation": "Method `MCPClientManager.get_client` calls `self._clients.get`; returns `self._clients.get(server_name)`.", "documented": true, "public": true, "addressable": true, "line": 162, "end_line": 172, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L162-L172", "decorators": [], "calls": [ "self._clients.get" ], "state_reads": [ "self._clients.get", "self._clients" ], "state_writes": [], "raises": [], "return_expressions": [ "self._clients.get(server_name)" ], "awaits": false, "yields": false }, { "name": "execute_tool", "qualname": "MCPClientManager.execute_tool", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.execute_tool", "kind": "method", "signature": "async def execute_tool(self, full_name: str, arguments: Dict[str, Any], timeout: Optional[float]=None) -> MCPToolResult", "parameters": [ { "name": "full_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Full tool name with server prefix" }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Tool arguments" }, { "name": "timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional timeout in seconds" } ], "return_annotation": "MCPToolResult", "docstring": "Execute a tool by its full name (server__tool).\n\nArgs:\n full_name: Full tool name with server prefix\n arguments: Tool arguments\n timeout: Optional timeout in seconds\n\nReturns:\n MCPToolResult with the result or error", "summary": "Execute a tool by its full name (server__tool).", "implementation": "Method `MCPClientManager.execute_tool` calls `openai_call_to_mcp`, `self._find_tool_server`, `MCPToolResult`, `self._clients.get`; awaits asynchronous work; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 174, "end_line": 232, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L174-L232", "decorators": [], "calls": [ "openai_call_to_mcp", "self._find_tool_server", "MCPToolResult", "self._clients.get", "client.call_tool" ], "state_reads": [ "self._find_tool_server", "self._clients.get", "self._clients", "self.config.default_timeout", "self.config" ], "state_writes": [], "raises": [], "return_expressions": [ "MCPToolResult(tool_name=full_name, content=None, is_error=True, error_message=f\"Tool '{full_name}' not found in any con…", "MCPToolResult(tool_name=full_name, content=None, is_error=True, error_message=f\"Server '{server_name}' not found\")", "MCPToolResult(tool_name=full_name, content=None, is_error=True, error_message=f\"Server '{server_name}' is not connected…", "await client.call_tool(tool_name, arguments, timeout=timeout or self.config.default_timeout)" ], "awaits": true, "yields": false }, { "name": "execute_tool_call", "qualname": "MCPClientManager.execute_tool_call", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.execute_tool_call", "kind": "method", "signature": "async def execute_tool_call(self, tool_call: Dict[str, Any], timeout: Optional[float]=None) -> MCPToolResult", "parameters": [ { "name": "tool_call", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "OpenAI tool call object" }, { "name": "timeout", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional timeout in seconds" } ], "return_annotation": "MCPToolResult", "docstring": "Execute a tool call from OpenAI format.\n\nArgs:\n tool_call: OpenAI tool call object\n timeout: Optional timeout in seconds\n\nReturns:\n MCPToolResult with the result or error", "summary": "Execute a tool call from OpenAI format.", "implementation": "Method `MCPClientManager.execute_tool_call` calls `openai_call_to_mcp`, `self.execute_tool`; awaits asynchronous work; returns `await self.execute_tool(full_name, arguments, timeout)`.", "documented": true, "public": true, "addressable": true, "line": 234, "end_line": 256, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L234-L256", "decorators": [], "calls": [ "openai_call_to_mcp", "self.execute_tool" ], "state_reads": [ "self.execute_tool" ], "state_writes": [], "raises": [], "return_expressions": [ "await self.execute_tool(full_name, arguments, timeout)" ], "awaits": true, "yields": false }, { "name": "_find_tool_server", "qualname": "MCPClientManager._find_tool_server", "full_name": "vllm_mlx.mcp.manager.MCPClientManager._find_tool_server", "kind": "method", "signature": "def _find_tool_server(self, tool_name: str) -> Optional[str]", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Tool name (without server prefix)" } ], "return_annotation": "Optional[str]", "docstring": "Find which server has a tool by name.\n\nArgs:\n tool_name: Tool name (without server prefix)\n\nReturns:\n Server name or None if not found", "summary": "Find which server has a tool by name.", "implementation": "Method `MCPClientManager._find_tool_server` calls `self._clients.values`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 258, "end_line": 273, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L258-L273", "decorators": [], "calls": [ "self._clients.values" ], "state_reads": [ "self._clients.values", "self._clients" ], "state_writes": [], "raises": [], "return_expressions": [ "client.name", "None" ], "awaits": false, "yields": false }, { "name": "refresh_tools", "qualname": "MCPClientManager.refresh_tools", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.refresh_tools", "kind": "method", "signature": "async def refresh_tools(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Refresh tools from all connected servers.", "summary": "Refresh tools from all connected servers.", "implementation": "Method `MCPClientManager.refresh_tools` calls `client.refresh_tools`, `self._clients.values`, `asyncio.gather`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 275, "end_line": 283, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L275-L283", "decorators": [], "calls": [ "client.refresh_tools", "self._clients.values", "asyncio.gather" ], "state_reads": [ "self._clients.values", "self._clients" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "reconnect", "qualname": "MCPClientManager.reconnect", "full_name": "vllm_mlx.mcp.manager.MCPClientManager.reconnect", "kind": "method", "signature": "async def reconnect(self, server_name: Optional[str]=None)", "parameters": [ { "name": "server_name", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Specific server to reconnect, or None for all" } ], "return_annotation": "not annotated", "docstring": "Reconnect to server(s).\n\nArgs:\n server_name: Specific server to reconnect, or None for all", "summary": "Reconnect to server(s).", "implementation": "Method `MCPClientManager.reconnect` calls `self._clients.get`, `client.disconnect`, `client.connect`, `self._clients.values`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 285, "end_line": 301, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/manager.py#L285-L301", "decorators": [], "calls": [ "self._clients.get", "client.disconnect", "client.connect", "self._clients.values" ], "state_reads": [ "self._clients.get", "self._clients", "self._clients.values" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false } ] }, { "name": "vllm_mlx.mcp.security", "path": "vllm_mlx/mcp/security.py", "page_path": "reference/api/vllm_mlx/mcp/security.md", "docstring": "MCP security module for command validation and sandboxing.\n\nThis module provides security controls to prevent command injection\nand other attacks via MCP server configurations.", "summary": "MCP security module for command validation and sandboxing.", "line_count": 852, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L1-L852", "members": [ "logger", "ALLOW_UNSAFE_ENV_VAR", "ALLOWED_COMMANDS", "DANGEROUS_PATTERNS", "DANGEROUS_ARG_PATTERNS", "BLOCKED_COMMAND_ARG_RULES", "CONTROL_CHARS", "MCPSecurityError", "MCPCommandValidator", "_validator", "get_validator", "set_validator", "validate_mcp_server_config", "DANGEROUS_TOOL_ARG_PATTERNS", "HIGH_RISK_TOOL_PATTERNS", "ToolExecutionAudit", "ToolSandbox", "_sandbox", "get_sandbox", "set_sandbox" ], "symbols": [ { "name": "MCPSecurityError", "qualname": "MCPSecurityError", "full_name": "vllm_mlx.mcp.security.MCPSecurityError", "kind": "class", "signature": "class MCPSecurityError(Exception)", "parameters": [], "return_annotation": "MCPSecurityError", "docstring": "Raised when MCP security validation fails.", "summary": "Raised when MCP security validation fails.", "implementation": "Class `MCPSecurityError` derives from `Exception` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 106, "end_line": 109, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L106-L109", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPCommandValidator", "qualname": "MCPCommandValidator", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator", "kind": "class", "signature": "class MCPCommandValidator", "parameters": [ { "name": "allowed_commands", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "Set of allowed command names. If None, uses default whitelist." }, { "name": "allow_unsafe", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "If True, allows any command (for development only)." }, { "name": "custom_whitelist", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "Additional commands to allow beyond the default whitelist." }, { "name": "check_path_exists", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, verify command exists in PATH. Set to False for testing." } ], "return_annotation": "MCPCommandValidator", "docstring": "Validates MCP server commands for security.\n\nThis class provides methods to validate commands and arguments\nbefore they are executed, preventing command injection attacks.", "summary": "Validates MCP server commands for security.", "implementation": "Class `MCPCommandValidator` declares 8 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 112, "end_line": 427, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L112-L427", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MCPCommandValidator.__init__", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator.__init__", "kind": "method", "signature": "def __init__(self, allowed_commands: Optional[Set[str]]=None, allow_unsafe: bool=False, custom_whitelist: Optional[Set[str]]=None, check_path_exists: bool=True)", "parameters": [ { "name": "allowed_commands", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "Set of allowed command names. If None, uses default whitelist." }, { "name": "allow_unsafe", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "If True, allows any command (for development only)." }, { "name": "custom_whitelist", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "Additional commands to allow beyond the default whitelist." }, { "name": "check_path_exists", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, verify command exists in PATH. Set to False for testing." } ], "return_annotation": "not annotated", "docstring": "Initialize the command validator.\n\nArgs:\n allowed_commands: Set of allowed command names. If None, uses default whitelist.\n allow_unsafe: If True, allows any command (for development only).\n WARNING: This disables security checks!\n custom_whitelist: Additional commands to allow beyond the default whitelist.\n check_path_exists: If True, verify command exists in PATH. Set to False for testing.", "summary": "Initialize the command validator.", "implementation": "Method `MCPCommandValidator.__init__` updates `self.allow_unsafe`, `self.allowed_commands`, `self.check_path_exists`; calls `ALLOWED_COMMANDS.copy`, `self.allowed_commands.update`, `logger.warning`.", "documented": true, "public": false, "addressable": true, "line": 120, "end_line": 149, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L120-L149", "decorators": [], "calls": [ "ALLOWED_COMMANDS.copy", "self.allowed_commands.update", "logger.warning" ], "state_reads": [ "self.allowed_commands.update", "self.allowed_commands" ], "state_writes": [ "self.allow_unsafe", "self.allowed_commands", "self.check_path_exists" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_check_control_chars", "qualname": "MCPCommandValidator._check_control_chars", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator._check_control_chars", "kind": "method", "signature": "def _check_control_chars(self, value: str, context: str, server_name: str) -> None", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "context", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Block command separators carried via literal newlines.", "summary": "Block command separators carried via literal newlines.", "implementation": "Method `MCPCommandValidator._check_control_chars` calls `any`, `MCPSecurityError`; can raise `MCPSecurityError`.", "documented": true, "public": false, "addressable": true, "line": 151, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L151-L157", "decorators": [], "calls": [ "any", "MCPSecurityError" ], "state_reads": [], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_check_path_traversal", "qualname": "MCPCommandValidator._check_path_traversal", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator._check_path_traversal", "kind": "method", "signature": "def _check_path_traversal(self, value: str, context: str, server_name: str) -> None", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "context", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Block parent-directory traversal, including URL-encoded forms.\n\nThis normalizes likely path-like inputs rather than relying only on\nthe simple ``../`` regex, which can be bypassed by percent-encoding.", "summary": "Block parent-directory traversal, including URL-encoded forms.", "implementation": "Method `MCPCommandValidator._check_path_traversal` calls `unquote`, `candidates.append`, `value.lower`, `posixpath.normpath`; can raise `MCPSecurityError`.", "documented": true, "public": false, "addressable": true, "line": 159, "end_line": 194, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L159-L194", "decorators": [], "calls": [ "unquote", "candidates.append", "value.lower", "posixpath.normpath", "candidate.replace", "normalized.startswith", "MCPSecurityError", "candidate.replace('\\\\', '/').split", "any" ], "state_reads": [], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "validate_command", "qualname": "MCPCommandValidator.validate_command", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator.validate_command", "kind": "method", "signature": "def validate_command(self, command: str, server_name: str) -> None", "parameters": [ { "name": "command", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The command to validate" }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the MCP server (for logging)" } ], "return_annotation": "None", "docstring": "Validate that a command is safe to execute.\n\nArgs:\n command: The command to validate\n server_name: Name of the MCP server (for logging)\n\nRaises:\n MCPSecurityError: If the command is not allowed", "summary": "Validate that a command is safe to execute.", "implementation": "Method `MCPCommandValidator.validate_command` calls `logger.warning`, `self._check_control_chars`, `self._check_path_traversal`, `pattern.search`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 196, "end_line": 258, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L196-L258", "decorators": [], "calls": [ "logger.warning", "self._check_control_chars", "self._check_path_traversal", "pattern.search", "MCPSecurityError", "Path", "os.path.isabs", "os.path.isfile", "os.access", "logger.info", "sorted", "shutil.which", "logger.debug" ], "state_reads": [ "self.allow_unsafe", "self._check_control_chars", "self._check_path_traversal", "self.allowed_commands", "self.check_path_exists" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "validate_args", "qualname": "MCPCommandValidator.validate_args", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator.validate_args", "kind": "method", "signature": "def validate_args(self, args: List[str], server_name: str) -> None", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "List of command arguments" }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the MCP server (for logging)" } ], "return_annotation": "None", "docstring": "Validate command arguments for dangerous patterns.\n\nArgs:\n args: List of command arguments\n server_name: Name of the MCP server (for logging)\n\nRaises:\n MCPSecurityError: If any argument contains dangerous patterns", "summary": "Validate command arguments for dangerous patterns.", "implementation": "Method `MCPCommandValidator.validate_args` calls `enumerate`, `self._check_control_chars`, `self._check_path_traversal`, `pattern.search`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 260, "end_line": 286, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L260-L286", "decorators": [], "calls": [ "enumerate", "self._check_control_chars", "self._check_path_traversal", "pattern.search", "MCPSecurityError", "logger.debug", "len" ], "state_reads": [ "self.allow_unsafe", "self._check_control_chars", "self._check_path_traversal" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "validate_command_args", "qualname": "MCPCommandValidator.validate_command_args", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator.validate_command_args", "kind": "method", "signature": "def validate_command_args(self, command: str, args: List[str], server_name: str) -> None", "parameters": [ { "name": "command", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "args", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Validate command-specific argument combinations.\n\nSome whitelisted runtimes (python, node, npx) remain acceptable for\nlaunching packaged MCP servers, but inline evaluator flags such as\n``python -c`` and ``node -e`` must be rejected.", "summary": "Validate command-specific argument combinations.", "implementation": "Method `MCPCommandValidator.validate_command_args` calls `Path`, `BLOCKED_COMMAND_ARG_RULES.get`, `enumerate`, `MCPSecurityError`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 288, "end_line": 330, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L288-L330", "decorators": [], "calls": [ "Path", "BLOCKED_COMMAND_ARG_RULES.get", "enumerate", "MCPSecurityError", "arg.startswith", "logger.debug" ], "state_reads": [ "self.allow_unsafe" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "validate_env", "qualname": "MCPCommandValidator.validate_env", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator.validate_env", "kind": "method", "signature": "def validate_env(self, env: Optional[Dict[str, str]], server_name: str) -> None", "parameters": [ { "name": "env", "kind": "positional or keyword", "annotation": "Optional[Dict[str, str]]", "default": "", "required": true, "description": "Dictionary of environment variables" }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the MCP server (for logging)" } ], "return_annotation": "None", "docstring": "Validate environment variables for dangerous values.\n\nArgs:\n env: Dictionary of environment variables\n server_name: Name of the MCP server (for logging)\n\nRaises:\n MCPSecurityError: If any env var contains dangerous patterns", "summary": "Validate environment variables for dangerous values.", "implementation": "Method `MCPCommandValidator.validate_env` calls `env.items`, `self._check_control_chars`, `self._check_path_traversal`, `key.upper`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 332, "end_line": 383, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L332-L383", "decorators": [], "calls": [ "env.items", "self._check_control_chars", "self._check_path_traversal", "key.upper", "MCPSecurityError", "pattern.search", "logger.debug", "len" ], "state_reads": [ "self.allow_unsafe", "self._check_control_chars", "self._check_path_traversal" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "validate_url", "qualname": "MCPCommandValidator.validate_url", "full_name": "vllm_mlx.mcp.security.MCPCommandValidator.validate_url", "kind": "method", "signature": "def validate_url(self, url: str, server_name: str) -> None", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The SSE URL to validate" }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the MCP server (for logging)" } ], "return_annotation": "None", "docstring": "Validate SSE URL for security.\n\nArgs:\n url: The SSE URL to validate\n server_name: Name of the MCP server (for logging)\n\nRaises:\n MCPSecurityError: If the URL is not safe", "summary": "Validate SSE URL for security.", "implementation": "Method `MCPCommandValidator.validate_url` calls `self._check_control_chars`, `url.startswith`, `MCPSecurityError`, `logger.warning`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 385, "end_line": 427, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L385-L427", "decorators": [], "calls": [ "self._check_control_chars", "url.startswith", "MCPSecurityError", "logger.warning", "urlparse", "self._check_path_traversal", "pattern.search", "logger.debug" ], "state_reads": [ "self.allow_unsafe", "self._check_control_chars", "self._check_path_traversal" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_validator", "qualname": "get_validator", "full_name": "vllm_mlx.mcp.security.get_validator", "kind": "function", "signature": "def get_validator() -> MCPCommandValidator", "parameters": [], "return_annotation": "MCPCommandValidator", "docstring": "Get the global command validator instance.", "summary": "Get the global command validator instance.", "implementation": "Function `get_validator` calls `MCPCommandValidator`, `os.environ.get`; returns `_validator`.", "documented": true, "public": true, "addressable": true, "line": 434, "end_line": 441, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L434-L441", "decorators": [], "calls": [ "MCPCommandValidator", "os.environ.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_validator" ], "awaits": false, "yields": false }, { "name": "set_validator", "qualname": "set_validator", "full_name": "vllm_mlx.mcp.security.set_validator", "kind": "function", "signature": "def set_validator(validator: MCPCommandValidator) -> None", "parameters": [ { "name": "validator", "kind": "positional or keyword", "annotation": "MCPCommandValidator", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Set a custom global validator.", "summary": "Set a custom global validator.", "implementation": "Function `set_validator` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 444, "end_line": 447, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L444-L447", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "validate_mcp_server_config", "qualname": "validate_mcp_server_config", "full_name": "vllm_mlx.mcp.security.validate_mcp_server_config", "kind": "function", "signature": "def validate_mcp_server_config(server_name: str, command: Optional[str]=None, args: Optional[List[str]]=None, env: Optional[Dict[str, str]]=None, url: Optional[str]=None) -> None", "parameters": [ { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the MCP server" }, { "name": "command", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Command to execute (for stdio transport)" }, { "name": "args", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Command arguments" }, { "name": "env", "kind": "positional or keyword", "annotation": "Optional[Dict[str, str]]", "default": "None", "required": false, "description": "Environment variables" }, { "name": "url", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "SSE URL (for sse transport)" } ], "return_annotation": "None", "docstring": "Validate MCP server configuration for security.\n\nThis is a convenience function that uses the global validator.\n\nArgs:\n server_name: Name of the MCP server\n command: Command to execute (for stdio transport)\n args: Command arguments\n env: Environment variables\n url: SSE URL (for sse transport)\n\nRaises:\n MCPSecurityError: If validation fails", "summary": "Validate MCP server configuration for security.", "implementation": "Function `validate_mcp_server_config` calls `get_validator`, `validator.validate_command`, `validator.validate_args`, `validator.validate_command_args`.", "documented": true, "public": true, "addressable": true, "line": 450, "end_line": 486, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L450-L486", "decorators": [], "calls": [ "get_validator", "validator.validate_command", "validator.validate_args", "validator.validate_command_args", "validator.validate_env", "validator.validate_url" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ToolExecutionAudit", "qualname": "ToolExecutionAudit", "full_name": "vllm_mlx.mcp.security.ToolExecutionAudit", "kind": "class", "signature": "class ToolExecutionAudit", "parameters": [ { "name": "timestamp", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tool_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "server_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "arguments", "kind": "field", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "success", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "error_message", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "execution_time_ms", "kind": "field", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ToolExecutionAudit", "docstring": "Record of a tool execution for audit purposes.", "summary": "Record of a tool execution for audit purposes.", "implementation": "Class `ToolExecutionAudit` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 516, "end_line": 525, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L516-L525", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ToolSandbox", "qualname": "ToolSandbox", "full_name": "vllm_mlx.mcp.security.ToolSandbox", "kind": "class", "signature": "class ToolSandbox", "parameters": [ { "name": "allowed_tools", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "If set, only these tools can be executed (whitelist mode)." }, { "name": "blocked_tools", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "Tools that are always blocked (blacklist mode)." }, { "name": "allowed_high_risk_tools", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "High-risk tools that are explicitly allowed." }, { "name": "blocked_arg_patterns", "kind": "positional or keyword", "annotation": "Optional[List[re.Pattern]]", "default": "None", "required": false, "description": "Patterns to block in tool arguments." }, { "name": "max_calls_per_minute", "kind": "positional or keyword", "annotation": "int", "default": "60", "required": false, "description": "Rate limit for tool calls (0 = unlimited)." }, { "name": "audit_callback", "kind": "positional or keyword", "annotation": "Optional[Callable[[ToolExecutionAudit], None]]", "default": "None", "required": false, "description": "Optional callback for audit events." }, { "name": "enabled", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If False, sandbox checks are bypassed (dev mode only)." } ], "return_annotation": "ToolSandbox", "docstring": "Sandboxing controls for MCP tool execution.\n\nProvides:\n- Tool allowlisting/blocklisting\n- Argument sanitization\n- Audit logging\n- Rate limiting", "summary": "Sandboxing controls for MCP tool execution.", "implementation": "Class `ToolSandbox` declares 10 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 528, "end_line": 834, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L528-L834", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ToolSandbox.__init__", "full_name": "vllm_mlx.mcp.security.ToolSandbox.__init__", "kind": "method", "signature": "def __init__(self, allowed_tools: Optional[Set[str]]=None, blocked_tools: Optional[Set[str]]=None, allowed_high_risk_tools: Optional[Set[str]]=None, blocked_arg_patterns: Optional[List[re.Pattern]]=None, max_calls_per_minute: int=60, audit_callback: Optional[Callable[[ToolExecutionAudit], None]]=None, enabled: bool=True)", "parameters": [ { "name": "allowed_tools", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "If set, only these tools can be executed (whitelist mode)." }, { "name": "blocked_tools", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "Tools that are always blocked (blacklist mode)." }, { "name": "allowed_high_risk_tools", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "High-risk tools that are explicitly allowed." }, { "name": "blocked_arg_patterns", "kind": "positional or keyword", "annotation": "Optional[List[re.Pattern]]", "default": "None", "required": false, "description": "Patterns to block in tool arguments." }, { "name": "max_calls_per_minute", "kind": "positional or keyword", "annotation": "int", "default": "60", "required": false, "description": "Rate limit for tool calls (0 = unlimited)." }, { "name": "audit_callback", "kind": "positional or keyword", "annotation": "Optional[Callable[[ToolExecutionAudit], None]]", "default": "None", "required": false, "description": "Optional callback for audit events." }, { "name": "enabled", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If False, sandbox checks are bypassed (dev mode only)." } ], "return_annotation": "not annotated", "docstring": "Initialize tool sandbox.\n\nArgs:\n allowed_tools: If set, only these tools can be executed (whitelist mode).\n blocked_tools: Tools that are always blocked (blacklist mode).\n allowed_high_risk_tools: High-risk tools that are explicitly allowed.\n blocked_arg_patterns: Patterns to block in tool arguments.\n max_calls_per_minute: Rate limit for tool calls (0 = unlimited).\n audit_callback: Optional callback for audit events.\n enabled: If False, sandbox checks are bypassed (dev mode only).", "summary": "Initialize tool sandbox.", "implementation": "Method `ToolSandbox.__init__` updates `self.allowed_tools`, `self.blocked_tools`, `self.allowed_high_risk_tools`, `self.blocked_arg_patterns`; calls `set`, `tool.lower`, `DANGEROUS_TOOL_ARG_PATTERNS.copy`, `defaultdict`.", "documented": true, "public": false, "addressable": true, "line": 539, "end_line": 586, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L539-L586", "decorators": [], "calls": [ "set", "tool.lower", "DANGEROUS_TOOL_ARG_PATTERNS.copy", "defaultdict", "Lock", "logger.warning" ], "state_reads": [], "state_writes": [ "self.allowed_tools", "self.blocked_tools", "self.allowed_high_risk_tools", "self.blocked_arg_patterns", "self.max_calls_per_minute", "self.audit_callback", "self.enabled", "self._call_times", "self._rate_limit_lock", "self._audit_log", "self._audit_log_max_size", "self._audit_lock" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "validate_tool_execution", "qualname": "ToolSandbox.validate_tool_execution", "full_name": "vllm_mlx.mcp.security.ToolSandbox.validate_tool_execution", "kind": "method", "signature": "def validate_tool_execution(self, tool_name: str, server_name: str, arguments: Dict[str, Any]) -> None", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the tool to execute" }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "MCP server providing the tool" }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Tool arguments" } ], "return_annotation": "None", "docstring": "Validate that a tool execution is allowed.\n\nArgs:\n tool_name: Name of the tool to execute\n server_name: MCP server providing the tool\n arguments: Tool arguments\n\nRaises:\n MCPSecurityError: If execution is not allowed", "summary": "Validate that a tool execution is allowed.", "implementation": "Method `ToolSandbox.validate_tool_execution` calls `logger.debug`, `self._is_blocked`, `MCPSecurityError`, `self._check_high_risk_tool`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 588, "end_line": 634, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L588-L634", "decorators": [], "calls": [ "logger.debug", "self._is_blocked", "MCPSecurityError", "self._check_high_risk_tool", "self._validate_arguments", "self._check_rate_limit" ], "state_reads": [ "self.enabled", "self._is_blocked", "self.allowed_tools", "self._check_high_risk_tool", "self._validate_arguments", "self._check_rate_limit" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_is_blocked", "qualname": "ToolSandbox._is_blocked", "full_name": "vllm_mlx.mcp.security.ToolSandbox._is_blocked", "kind": "method", "signature": "def _is_blocked(self, tool_name: str, full_name: str) -> bool", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "full_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if tool is in blocklist.", "summary": "Check if tool is in blocklist.", "implementation": "Method `ToolSandbox._is_blocked` calls `tool_name.lower`; returns `tool_name in self.blocked_tools or full_name in self.blocked_tools or tool_name.lower() in self.blocked_tools`.", "documented": true, "public": false, "addressable": true, "line": 636, "end_line": 642, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L636-L642", "decorators": [], "calls": [ "tool_name.lower" ], "state_reads": [ "self.blocked_tools" ], "state_writes": [], "raises": [], "return_expressions": [ "tool_name in self.blocked_tools or full_name in self.blocked_tools or tool_name.lower() in self.blocked_tools" ], "awaits": false, "yields": false }, { "name": "_check_high_risk_tool", "qualname": "ToolSandbox._check_high_risk_tool", "full_name": "vllm_mlx.mcp.security.ToolSandbox._check_high_risk_tool", "kind": "method", "signature": "def _check_high_risk_tool(self, tool_name: str, full_name: str) -> None", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "full_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Check if tool matches high-risk patterns.", "summary": "Check if tool matches high-risk patterns.", "implementation": "Method `ToolSandbox._check_high_risk_tool` calls `tool_name.lower`, `full_name.lower`, `logger.warning`, `MCPSecurityError`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 644, "end_line": 663, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L644-L663", "decorators": [], "calls": [ "tool_name.lower", "full_name.lower", "logger.warning", "MCPSecurityError" ], "state_reads": [ "self.allowed_high_risk_tools" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_validate_arguments", "qualname": "ToolSandbox._validate_arguments", "full_name": "vllm_mlx.mcp.security.ToolSandbox._validate_arguments", "kind": "method", "signature": "def _validate_arguments(self, tool_name: str, arguments: Dict[str, Any]) -> None", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Validate tool arguments for dangerous patterns.", "summary": "Validate tool arguments for dangerous patterns.", "implementation": "Method `ToolSandbox._validate_arguments` calls `arguments.items`, `check_value`.", "documented": true, "public": false, "addressable": true, "line": 665, "end_line": 686, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L665-L686", "decorators": [], "calls": [ "arguments.items", "check_value" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "check_value", "qualname": "ToolSandbox._validate_arguments.check_value", "full_name": "vllm_mlx.mcp.security.ToolSandbox._validate_arguments.check_value", "kind": "nested function", "signature": "def check_value(key: str, value: Any, path: str='') -> None", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "path", "kind": "positional or keyword", "annotation": "str", "default": "''", "required": false, "description": "Optional positional or keyword input; defaults to `''`." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `ToolSandbox._validate_arguments.check_value` calls `isinstance`, `pattern.search`, `MCPSecurityError`, `value.items`; can raise `MCPSecurityError`.", "implementation": "Nested Function `ToolSandbox._validate_arguments.check_value` calls `isinstance`, `pattern.search`, `MCPSecurityError`, `value.items`; can raise `MCPSecurityError`.", "documented": false, "public": false, "addressable": false, "line": 668, "end_line": 683, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L668-L683", "decorators": [], "calls": [ "isinstance", "pattern.search", "MCPSecurityError", "value.items", "check_value", "enumerate" ], "state_reads": [ "self.blocked_arg_patterns" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_check_rate_limit", "qualname": "ToolSandbox._check_rate_limit", "full_name": "vllm_mlx.mcp.security.ToolSandbox._check_rate_limit", "kind": "method", "signature": "def _check_rate_limit(self, full_name: str) -> None", "parameters": [ { "name": "full_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Check and enforce rate limit for tool calls.", "summary": "Check and enforce rate limit for tool calls.", "implementation": "Method `ToolSandbox._check_rate_limit` calls `time.time`, `len`, `MCPSecurityError`, `self._call_times[full_name].append`; can raise `MCPSecurityError`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 688, "end_line": 710, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L688-L710", "decorators": [], "calls": [ "time.time", "len", "MCPSecurityError", "self._call_times[full_name].append" ], "state_reads": [ "self.max_calls_per_minute", "self._rate_limit_lock", "self._call_times" ], "state_writes": [], "raises": [ "MCPSecurityError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "record_execution", "qualname": "ToolSandbox.record_execution", "full_name": "vllm_mlx.mcp.security.ToolSandbox.record_execution", "kind": "method", "signature": "def record_execution(self, tool_name: str, server_name: str, arguments: Dict[str, Any], success: bool, error_message: Optional[str]=None, execution_time_ms: Optional[float]=None) -> ToolExecutionAudit", "parameters": [ { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the executed tool" }, { "name": "server_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "MCP server that executed the tool" }, { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Arguments passed to the tool" }, { "name": "success", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Whether execution succeeded" }, { "name": "error_message", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Error message if failed" }, { "name": "execution_time_ms", "kind": "positional or keyword", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Execution time in milliseconds" } ], "return_annotation": "ToolExecutionAudit", "docstring": "Record a tool execution for audit purposes.\n\nArgs:\n tool_name: Name of the executed tool\n server_name: MCP server that executed the tool\n arguments: Arguments passed to the tool\n success: Whether execution succeeded\n error_message: Error message if failed\n execution_time_ms: Execution time in milliseconds\n\nReturns:\n The audit record", "summary": "Record a tool execution for audit purposes.", "implementation": "Method `ToolSandbox.record_execution` updates `self._audit_log`; calls `ToolExecutionAudit`, `time.time`, `self._sanitize_arguments_for_log`, `self._audit_log.append`; returns `audit`.", "documented": true, "public": true, "addressable": true, "line": 712, "end_line": 772, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L712-L772", "decorators": [], "calls": [ "ToolExecutionAudit", "time.time", "self._sanitize_arguments_for_log", "self._audit_log.append", "len", "logger.info", "logger.warning", "self.audit_callback", "logger.error" ], "state_reads": [ "self._sanitize_arguments_for_log", "self._audit_lock", "self._audit_log.append", "self._audit_log", "self._audit_log_max_size", "self.audit_callback" ], "state_writes": [ "self._audit_log" ], "raises": [], "return_expressions": [ "audit" ], "awaits": false, "yields": false }, { "name": "_sanitize_arguments_for_log", "qualname": "ToolSandbox._sanitize_arguments_for_log", "full_name": "vllm_mlx.mcp.security.ToolSandbox._sanitize_arguments_for_log", "kind": "method", "signature": "def _sanitize_arguments_for_log(self, arguments: Dict[str, Any]) -> Dict[str, Any]", "parameters": [ { "name": "arguments", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Dict[str, Any]", "docstring": "Sanitize arguments for logging (redact sensitive data).", "summary": "Sanitize arguments for logging (redact sensitive data).", "implementation": "Method `ToolSandbox._sanitize_arguments_for_log` calls `sanitize`; returns `sanitize(arguments)`.", "documented": true, "public": false, "addressable": true, "line": 774, "end_line": 794, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L774-L794", "decorators": [], "calls": [ "sanitize" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "sanitize(arguments)" ], "awaits": false, "yields": false }, { "name": "sanitize", "qualname": "ToolSandbox._sanitize_arguments_for_log.sanitize", "full_name": "vllm_mlx.mcp.security.ToolSandbox._sanitize_arguments_for_log.sanitize", "kind": "nested function", "signature": "def sanitize(obj: Any) -> Any", "parameters": [ { "name": "obj", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "", "summary": "Nested Function `ToolSandbox._sanitize_arguments_for_log.sanitize` calls `isinstance`, `any`, `k.lower`, `sanitize`; has 4 explicit return paths.", "implementation": "Nested Function `ToolSandbox._sanitize_arguments_for_log.sanitize` calls `isinstance`, `any`, `k.lower`, `sanitize`; has 4 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 778, "end_line": 792, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L778-L792", "decorators": [], "calls": [ "isinstance", "any", "k.lower", "sanitize", "obj.items", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{k: '[REDACTED]' if any((s in k.lower() for s in sensitive_keys)) else sanitize(v) for k, v in obj.items()}", "[sanitize(item) for item in obj]", "obj[:100] + f'... [truncated, {len(obj)} chars total]'", "obj" ], "awaits": false, "yields": false }, { "name": "get_audit_log", "qualname": "ToolSandbox.get_audit_log", "full_name": "vllm_mlx.mcp.security.ToolSandbox.get_audit_log", "kind": "method", "signature": "def get_audit_log(self, limit: int=100, tool_filter: Optional[str]=None, server_filter: Optional[str]=None, errors_only: bool=False) -> List[ToolExecutionAudit]", "parameters": [ { "name": "limit", "kind": "positional or keyword", "annotation": "int", "default": "100", "required": false, "description": "Maximum entries to return" }, { "name": "tool_filter", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Filter by tool name (substring match)" }, { "name": "server_filter", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Filter by server name" }, { "name": "errors_only", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Only return failed executions" } ], "return_annotation": "List[ToolExecutionAudit]", "docstring": "Get audit log entries.\n\nArgs:\n limit: Maximum entries to return\n tool_filter: Filter by tool name (substring match)\n server_filter: Filter by server name\n errors_only: Only return failed executions\n\nReturns:\n List of audit entries", "summary": "Get audit log entries.", "implementation": "Method `ToolSandbox.get_audit_log` calls `self._audit_log.copy`; returns `entries[-limit:]`.", "documented": true, "public": true, "addressable": true, "line": 796, "end_line": 827, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L796-L827", "decorators": [], "calls": [ "self._audit_log.copy" ], "state_reads": [ "self._audit_lock", "self._audit_log.copy", "self._audit_log" ], "state_writes": [], "raises": [], "return_expressions": [ "entries[-limit:]" ], "awaits": false, "yields": false }, { "name": "clear_audit_log", "qualname": "ToolSandbox.clear_audit_log", "full_name": "vllm_mlx.mcp.security.ToolSandbox.clear_audit_log", "kind": "method", "signature": "def clear_audit_log(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Clear audit log and return number of entries cleared.", "summary": "Clear audit log and return number of entries cleared.", "implementation": "Method `ToolSandbox.clear_audit_log` calls `len`, `self._audit_log.clear`; returns `count`.", "documented": true, "public": true, "addressable": true, "line": 829, "end_line": 834, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L829-L834", "decorators": [], "calls": [ "len", "self._audit_log.clear" ], "state_reads": [ "self._audit_lock", "self._audit_log", "self._audit_log.clear" ], "state_writes": [], "raises": [], "return_expressions": [ "count" ], "awaits": false, "yields": false }, { "name": "get_sandbox", "qualname": "get_sandbox", "full_name": "vllm_mlx.mcp.security.get_sandbox", "kind": "function", "signature": "def get_sandbox() -> ToolSandbox", "parameters": [], "return_annotation": "ToolSandbox", "docstring": "Get the global tool sandbox instance.", "summary": "Get the global tool sandbox instance.", "implementation": "Function `get_sandbox` calls `ToolSandbox`; returns `_sandbox`.", "documented": true, "public": true, "addressable": true, "line": 841, "end_line": 846, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L841-L846", "decorators": [], "calls": [ "ToolSandbox" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_sandbox" ], "awaits": false, "yields": false }, { "name": "set_sandbox", "qualname": "set_sandbox", "full_name": "vllm_mlx.mcp.security.set_sandbox", "kind": "function", "signature": "def set_sandbox(sandbox: ToolSandbox) -> None", "parameters": [ { "name": "sandbox", "kind": "positional or keyword", "annotation": "ToolSandbox", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Set a custom global sandbox.", "summary": "Set a custom global sandbox.", "implementation": "Function `set_sandbox` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 849, "end_line": 852, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/security.py#L849-L852", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.mcp.tools", "path": "vllm_mlx/mcp/tools.py", "page_path": "reference/api/vllm_mlx/mcp/tools.md", "docstring": "Tool schema conversion utilities for MCP <-> OpenAI formats.", "summary": "Tool schema conversion utilities for MCP <-> OpenAI formats.", "line_count": 174, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L1-L174", "members": [ "mcp_tool_to_openai", "mcp_tools_to_openai", "openai_call_to_mcp", "format_tool_result", "format_tool_results", "merge_tools", "extract_tool_calls", "has_tool_calls" ], "symbols": [ { "name": "mcp_tool_to_openai", "qualname": "mcp_tool_to_openai", "full_name": "vllm_mlx.mcp.tools.mcp_tool_to_openai", "kind": "function", "signature": "def mcp_tool_to_openai(tool: MCPTool) -> Dict[str, Any]", "parameters": [ { "name": "tool", "kind": "positional or keyword", "annotation": "MCPTool", "default": "", "required": true, "description": "MCPTool instance" } ], "return_annotation": "Dict[str, Any]", "docstring": "Convert MCP tool schema to OpenAI function calling format.\n\nArgs:\n tool: MCPTool instance\n\nReturns:\n OpenAI-compatible tool definition", "summary": "Convert MCP tool schema to OpenAI function calling format.", "implementation": "Function `mcp_tool_to_openai` returns `{'type': 'function', 'function': {'name': tool.full_name, 'description': tool.description, 'parameters': tool.input_sch…`.", "documented": true, "public": true, "addressable": true, "line": 12, "end_line": 33, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L12-L33", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'type': 'function', 'function': {'name': tool.full_name, 'description': tool.description, 'parameters': tool.input_sch…" ], "awaits": false, "yields": false }, { "name": "mcp_tools_to_openai", "qualname": "mcp_tools_to_openai", "full_name": "vllm_mlx.mcp.tools.mcp_tools_to_openai", "kind": "function", "signature": "def mcp_tools_to_openai(tools: List[MCPTool]) -> List[Dict[str, Any]]", "parameters": [ { "name": "tools", "kind": "positional or keyword", "annotation": "List[MCPTool]", "default": "", "required": true, "description": "List of MCPTool instances" } ], "return_annotation": "List[Dict[str, Any]]", "docstring": "Convert list of MCP tools to OpenAI format.\n\nArgs:\n tools: List of MCPTool instances\n\nReturns:\n List of OpenAI-compatible tool definitions", "summary": "Convert list of MCP tools to OpenAI format.", "implementation": "Function `mcp_tools_to_openai` calls `mcp_tool_to_openai`; returns `[mcp_tool_to_openai(tool) for tool in tools]`.", "documented": true, "public": true, "addressable": true, "line": 36, "end_line": 46, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L36-L46", "decorators": [], "calls": [ "mcp_tool_to_openai" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[mcp_tool_to_openai(tool) for tool in tools]" ], "awaits": false, "yields": false }, { "name": "openai_call_to_mcp", "qualname": "openai_call_to_mcp", "full_name": "vllm_mlx.mcp.tools.openai_call_to_mcp", "kind": "function", "signature": "def openai_call_to_mcp(tool_call: Dict[str, Any]) -> Tuple[str, str, Dict[str, Any]]", "parameters": [ { "name": "tool_call", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "OpenAI tool call from model response" } ], "return_annotation": "Tuple[str, str, Dict[str, Any]]", "docstring": "Parse OpenAI tool call back to MCP format.\n\nArgs:\n tool_call: OpenAI tool call from model response\n\nReturns:\n Tuple of (server_name, tool_name, arguments)\n\nRaises:\n ValueError: If tool call format is invalid", "summary": "Parse OpenAI tool call back to MCP format.", "implementation": "Function `openai_call_to_mcp` calls `tool_call.get`, `function.get`, `isinstance`, `json.loads`; returns `(server_name, tool_name, arguments)`.", "documented": true, "public": true, "addressable": true, "line": 49, "end_line": 84, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L49-L84", "decorators": [], "calls": [ "tool_call.get", "function.get", "isinstance", "json.loads", "full_name.split" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(server_name, tool_name, arguments)" ], "awaits": false, "yields": false }, { "name": "format_tool_result", "qualname": "format_tool_result", "full_name": "vllm_mlx.mcp.tools.format_tool_result", "kind": "function", "signature": "def format_tool_result(result: MCPToolResult, tool_call_id: str) -> Dict[str, Any]", "parameters": [ { "name": "result", "kind": "positional or keyword", "annotation": "MCPToolResult", "default": "", "required": true, "description": "MCPToolResult from tool execution" }, { "name": "tool_call_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "ID of the tool call this is responding to" } ], "return_annotation": "Dict[str, Any]", "docstring": "Format tool result for inclusion in conversation messages.\n\nArgs:\n result: MCPToolResult from tool execution\n tool_call_id: ID of the tool call this is responding to\n\nReturns:\n OpenAI-compatible tool result message", "summary": "Format tool result for inclusion in conversation messages.", "implementation": "Function `format_tool_result` calls `result.to_message`; returns `result.to_message(tool_call_id)`.", "documented": true, "public": true, "addressable": true, "line": 87, "end_line": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L87-L98", "decorators": [], "calls": [ "result.to_message" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result.to_message(tool_call_id)" ], "awaits": false, "yields": false }, { "name": "format_tool_results", "qualname": "format_tool_results", "full_name": "vllm_mlx.mcp.tools.format_tool_results", "kind": "function", "signature": "def format_tool_results(results: List[Tuple[MCPToolResult, str]]) -> List[Dict[str, Any]]", "parameters": [ { "name": "results", "kind": "positional or keyword", "annotation": "List[Tuple[MCPToolResult, str]]", "default": "", "required": true, "description": "List of (MCPToolResult, tool_call_id) tuples" } ], "return_annotation": "List[Dict[str, Any]]", "docstring": "Format multiple tool results as messages.\n\nArgs:\n results: List of (MCPToolResult, tool_call_id) tuples\n\nReturns:\n List of OpenAI-compatible tool result messages", "summary": "Format multiple tool results as messages.", "implementation": "Function `format_tool_results` calls `format_tool_result`; returns `[format_tool_result(result, call_id) for result, call_id in results]`.", "documented": true, "public": true, "addressable": true, "line": 101, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L101-L113", "decorators": [], "calls": [ "format_tool_result" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[format_tool_result(result, call_id) for result, call_id in results]" ], "awaits": false, "yields": false }, { "name": "merge_tools", "qualname": "merge_tools", "full_name": "vllm_mlx.mcp.tools.merge_tools", "kind": "function", "signature": "def merge_tools(mcp_tools: List[MCPTool], user_tools: Optional[List[Dict[str, Any]]]=None) -> List[Dict[str, Any]]", "parameters": [ { "name": "mcp_tools", "kind": "positional or keyword", "annotation": "List[MCPTool]", "default": "", "required": true, "description": "Tools discovered from MCP servers" }, { "name": "user_tools", "kind": "positional or keyword", "annotation": "Optional[List[Dict[str, Any]]]", "default": "None", "required": false, "description": "User-provided tools in OpenAI format" } ], "return_annotation": "List[Dict[str, Any]]", "docstring": "Merge MCP tools with user-provided tools.\n\nUser tools take precedence if there are name conflicts.\n\nArgs:\n mcp_tools: Tools discovered from MCP servers\n user_tools: User-provided tools in OpenAI format\n\nReturns:\n Combined list of tools in OpenAI format", "summary": "Merge MCP tools with user-provided tools.", "implementation": "Function `merge_tools` calls `mcp_tool_to_openai`, `tool.get`, `func.get`, `list`; returns `list(all_tools.values())`.", "documented": true, "public": true, "addressable": true, "line": 116, "end_line": 143, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L116-L143", "decorators": [], "calls": [ "mcp_tool_to_openai", "tool.get", "func.get", "list", "all_tools.values" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "list(all_tools.values())" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "extract_tool_calls", "full_name": "vllm_mlx.mcp.tools.extract_tool_calls", "kind": "function", "signature": "def extract_tool_calls(response: Dict[str, Any]) -> List[Dict[str, Any]]", "parameters": [ { "name": "response", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "OpenAI-format model response" } ], "return_annotation": "List[Dict[str, Any]]", "docstring": "Extract tool calls from model response.\n\nArgs:\n response: OpenAI-format model response\n\nReturns:\n List of tool calls", "summary": "Extract tool calls from model response.", "implementation": "Function `extract_tool_calls` calls `response.get`, `choices[0].get`, `message.get`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 146, "end_line": 161, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L146-L161", "decorators": [], "calls": [ "response.get", "choices[0].get", "message.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "message.get('tool_calls', [])" ], "awaits": false, "yields": false }, { "name": "has_tool_calls", "qualname": "has_tool_calls", "full_name": "vllm_mlx.mcp.tools.has_tool_calls", "kind": "function", "signature": "def has_tool_calls(response: Dict[str, Any]) -> bool", "parameters": [ { "name": "response", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "OpenAI-format model response" } ], "return_annotation": "bool", "docstring": "Check if response contains tool calls.\n\nArgs:\n response: OpenAI-format model response\n\nReturns:\n True if response contains tool calls", "summary": "Check if response contains tool calls.", "implementation": "Function `has_tool_calls` calls `len`, `extract_tool_calls`; returns `len(extract_tool_calls(response)) > 0`.", "documented": true, "public": true, "addressable": true, "line": 164, "end_line": 174, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/tools.py#L164-L174", "decorators": [], "calls": [ "len", "extract_tool_calls" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "len(extract_tool_calls(response)) > 0" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.mcp.types", "path": "vllm_mlx/mcp/types.py", "page_path": "reference/api/vllm_mlx/mcp/types.md", "docstring": "Type definitions for MCP client support.", "summary": "Type definitions for MCP client support.", "line_count": 179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L1-L179", "members": [ "MCPTransport", "MCPServerState", "MCPServerConfig", "MCPConfig", "MCPTool", "MCPToolResult", "MCPServerStatus" ], "symbols": [ { "name": "MCPTransport", "qualname": "MCPTransport", "full_name": "vllm_mlx.mcp.types.MCPTransport", "kind": "class", "signature": "class MCPTransport(str, Enum)", "parameters": [], "return_annotation": "MCPTransport", "docstring": "Supported MCP transport types.", "summary": "Supported MCP transport types.", "implementation": "Class `MCPTransport` derives from `str`, `Enum` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 11, "end_line": 15, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L11-L15", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPServerState", "qualname": "MCPServerState", "full_name": "vllm_mlx.mcp.types.MCPServerState", "kind": "class", "signature": "class MCPServerState(str, Enum)", "parameters": [], "return_annotation": "MCPServerState", "docstring": "MCP server connection states.", "summary": "MCP server connection states.", "implementation": "Class `MCPServerState` derives from `str`, `Enum` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 18, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L18-L24", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPServerConfig", "qualname": "MCPServerConfig", "full_name": "vllm_mlx.mcp.types.MCPServerConfig", "kind": "class", "signature": "class MCPServerConfig", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "transport", "kind": "field", "annotation": "MCPTransport", "default": "MCPTransport.STDIO", "required": false, "description": "Optional constructor field; defaults to `MCPTransport.STDIO`." }, { "name": "command", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "args", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "env", "kind": "field", "annotation": "Optional[Dict[str, str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "url", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "enabled", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "timeout", "kind": "field", "annotation": "float", "default": "30.0", "required": false, "description": "Optional constructor field; defaults to `30.0`." } ], "return_annotation": "MCPServerConfig", "docstring": "Configuration for a single MCP server.", "summary": "Configuration for a single MCP server.", "implementation": "Class `MCPServerConfig` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L28-L78", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__post_init__", "qualname": "MCPServerConfig.__post_init__", "full_name": "vllm_mlx.mcp.types.MCPServerConfig.__post_init__", "kind": "method", "signature": "def __post_init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Validate configuration.", "summary": "Validate configuration.", "implementation": "Method `MCPServerConfig.__post_init__` updates `self.transport`; calls `isinstance`, `MCPTransport`, `ValueError`, `self._validate_security`; can raise `ValueError`.", "documented": true, "public": false, "addressable": true, "line": 46, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L46-L63", "decorators": [], "calls": [ "isinstance", "MCPTransport", "ValueError", "self._validate_security" ], "state_reads": [ "self.transport", "self.command", "self.name", "self.url", "self._validate_security" ], "state_writes": [ "self.transport" ], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_validate_security", "qualname": "MCPServerConfig._validate_security", "full_name": "vllm_mlx.mcp.types.MCPServerConfig._validate_security", "kind": "method", "signature": "def _validate_security(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Validate security of the configuration.", "summary": "Validate security of the configuration.", "implementation": "Method `MCPServerConfig._validate_security` calls `validate_mcp_server_config`, `ValueError`, `str`; can raise `ValueError`.", "documented": true, "public": false, "addressable": true, "line": 65, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L65-L78", "decorators": [], "calls": [ "validate_mcp_server_config", "ValueError", "str" ], "state_reads": [ "self.name", "self.command", "self.args", "self.env", "self.url" ], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MCPConfig", "qualname": "MCPConfig", "full_name": "vllm_mlx.mcp.types.MCPConfig", "kind": "class", "signature": "class MCPConfig", "parameters": [ { "name": "servers", "kind": "field", "annotation": "Dict[str, MCPServerConfig]", "default": "field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=dict)`." }, { "name": "max_tool_calls", "kind": "field", "annotation": "int", "default": "10", "required": false, "description": "Optional constructor field; defaults to `10`." }, { "name": "default_timeout", "kind": "field", "annotation": "float", "default": "30.0", "required": false, "description": "Optional constructor field; defaults to `30.0`." }, { "name": "allowed_high_risk_tools", "kind": "field", "annotation": "Set[str]", "default": "field(default_factory=set)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=set)`." } ], "return_annotation": "MCPConfig", "docstring": "Root configuration for MCP client.", "summary": "Root configuration for MCP client.", "implementation": "Class `MCPConfig` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 82, "end_line": 103, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L82-L103", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "from_dict", "qualname": "MCPConfig.from_dict", "full_name": "vllm_mlx.mcp.types.MCPConfig.from_dict", "kind": "method", "signature": "def from_dict(cls, data: Dict[str, Any]) -> 'MCPConfig'", "parameters": [ { "name": "data", "kind": "positional or keyword", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "'MCPConfig'", "docstring": "Create config from dictionary.", "summary": "Create config from dictionary.", "implementation": "Method `MCPConfig.from_dict` calls `data.get('servers', {}).items`, `data.get`, `MCPServerConfig`, `cls`; returns `cls(servers=servers, max_tool_calls=data.get('max_tool_calls', 10), default_timeout=data.get('default_timeout', 30.0), …`.", "documented": true, "public": true, "addressable": true, "line": 91, "end_line": 103, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L91-L103", "decorators": [ "classmethod" ], "calls": [ "data.get('servers', {}).items", "data.get", "MCPServerConfig", "cls", "set" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cls(servers=servers, max_tool_calls=data.get('max_tool_calls', 10), default_timeout=data.get('default_timeout', 30.0), …" ], "awaits": false, "yields": false }, { "name": "MCPTool", "qualname": "MCPTool", "full_name": "vllm_mlx.mcp.types.MCPTool", "kind": "class", "signature": "class MCPTool", "parameters": [ { "name": "server_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "description", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "input_schema", "kind": "field", "annotation": "Dict[str, Any]", "default": "field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=dict)`." } ], "return_annotation": "MCPTool", "docstring": "Normalized tool representation from MCP server.", "summary": "Normalized tool representation from MCP server.", "implementation": "Class `MCPTool` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 107, "end_line": 129, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L107-L129", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "full_name", "qualname": "MCPTool.full_name", "full_name": "vllm_mlx.mcp.types.MCPTool.full_name", "kind": "method", "signature": "def full_name(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Get namespaced tool name (server__tool).", "summary": "Get namespaced tool name (server__tool).", "implementation": "Method `MCPTool.full_name` returns `f'{self.server_name}__{self.name}'`.", "documented": true, "public": true, "addressable": true, "line": 116, "end_line": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L116-L118", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.server_name", "self.name" ], "state_writes": [], "raises": [], "return_expressions": [ "f'{self.server_name}__{self.name}'" ], "awaits": false, "yields": false }, { "name": "to_openai_format", "qualname": "MCPTool.to_openai_format", "full_name": "vllm_mlx.mcp.types.MCPTool.to_openai_format", "kind": "method", "signature": "def to_openai_format(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Convert to OpenAI function calling format.", "summary": "Convert to OpenAI function calling format.", "implementation": "Method `MCPTool.to_openai_format` returns `{'type': 'function', 'function': {'name': self.full_name, 'description': self.description, 'parameters': self.input_sch…`.", "documented": true, "public": true, "addressable": true, "line": 120, "end_line": 129, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L120-L129", "decorators": [], "calls": [], "state_reads": [ "self.full_name", "self.description", "self.input_schema" ], "state_writes": [], "raises": [], "return_expressions": [ "{'type': 'function', 'function': {'name': self.full_name, 'description': self.description, 'parameters': self.input_sch…" ], "awaits": false, "yields": false }, { "name": "MCPToolResult", "qualname": "MCPToolResult", "full_name": "vllm_mlx.mcp.types.MCPToolResult", "kind": "class", "signature": "class MCPToolResult", "parameters": [ { "name": "tool_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "content", "kind": "field", "annotation": "Any", "default": "", "required": true, "description": "Required constructor field." }, { "name": "is_error", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "error_message", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MCPToolResult", "docstring": "Result from a tool execution.", "summary": "Result from a tool execution.", "implementation": "Class `MCPToolResult` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 133, "end_line": 156, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L133-L156", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_message", "qualname": "MCPToolResult.to_message", "full_name": "vllm_mlx.mcp.types.MCPToolResult.to_message", "kind": "method", "signature": "def to_message(self, tool_call_id: str) -> Dict[str, Any]", "parameters": [ { "name": "tool_call_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Dict[str, Any]", "docstring": "Convert to OpenAI tool result message format.", "summary": "Convert to OpenAI tool result message format.", "implementation": "Method `MCPToolResult.to_message` calls `isinstance`, `json.dumps`; returns `{'role': 'tool', 'tool_call_id': tool_call_id, 'content': content}`.", "documented": true, "public": true, "addressable": true, "line": 141, "end_line": 156, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L141-L156", "decorators": [], "calls": [ "isinstance", "json.dumps" ], "state_reads": [ "self.is_error", "self.error_message", "self.content" ], "state_writes": [], "raises": [], "return_expressions": [ "{'role': 'tool', 'tool_call_id': tool_call_id, 'content': content}" ], "awaits": false, "yields": false }, { "name": "MCPServerStatus", "qualname": "MCPServerStatus", "full_name": "vllm_mlx.mcp.types.MCPServerStatus", "kind": "class", "signature": "class MCPServerStatus", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "state", "kind": "field", "annotation": "MCPServerState", "default": "", "required": true, "description": "Required constructor field." }, { "name": "transport", "kind": "field", "annotation": "MCPTransport", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tools_count", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "error", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "last_connected", "kind": "field", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MCPServerStatus", "docstring": "Status of an MCP server connection.", "summary": "Status of an MCP server connection.", "implementation": "Class `MCPServerStatus` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 160, "end_line": 179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L160-L179", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "MCPServerStatus.to_dict", "full_name": "vllm_mlx.mcp.types.MCPServerStatus.to_dict", "kind": "method", "signature": "def to_dict(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Convert to dictionary for API response.", "summary": "Convert to dictionary for API response.", "implementation": "Method `MCPServerStatus.to_dict` returns `{'name': self.name, 'state': self.state.value, 'transport': self.transport.value, 'tools_count': self.tools_count, 'err…`.", "documented": true, "public": true, "addressable": true, "line": 170, "end_line": 179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mcp/types.py#L170-L179", "decorators": [], "calls": [], "state_reads": [ "self.name", "self.state.value", "self.state", "self.transport.value", "self.transport", "self.tools_count", "self.error", "self.last_connected" ], "state_writes": [], "raises": [], "return_expressions": [ "{'name': self.name, 'state': self.state.value, 'transport': self.transport.value, 'tools_count': self.tools_count, 'err…" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.memory_cache", "path": "vllm_mlx/memory_cache.py", "page_path": "reference/api/vllm_mlx/memory_cache.md", "docstring": "Memory-aware prefix cache for vllm-mlx.\n\nThis module provides a prefix cache implementation that tracks memory usage\nand evicts entries based on memory pressure rather than entry count.\n\nKey features:\n- Automatic memory limit detection based on available system RAM\n- Accurate memory tracking for MLX array caches\n- LRU eviction triggered by memory thresholds\n- No unnecessary deep copies (MLX arrays are immutable)\n\nExample:\n config = MemoryCacheConfig(max_memory_percent=0.25)\n cache = MemoryAwarePrefixCache(model, config)\n\n # Fetch returns reference (no copy) - safe because MLX arrays are immutable\n kv_cache, remaining = cache.fetch(tokens)\n\n # Store tracks memory automatically\n cache.store(tokens, kv_cache)", "summary": "Memory-aware prefix cache for vllm-mlx.", "line_count": 1463, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1-L1463", "members": [ "logger", "_BYTES_PER_MB", "_DEFAULT_MEMORY_PERCENT", "_MIN_MEMORY_BYTES", "_MAX_ENTRIES_FALLBACK", "_CACHE_PERSIST_VERSION", "_get_available_memory", "_array_memory", "_nested_array_memory", "estimate_kv_cache_memory", "MemoryCacheConfig", "CacheStats", "_CacheEntry", "_is_cache_layer_trimmable", "_trim_cache_offset", "_needs_kv_trim", "_trim_to_offset", "_QuantizedCacheWrapper", "_quantize_cache", "_dequantize_cache", "_compute_model_fingerprint", "MemoryAwarePrefixCache" ], "symbols": [ { "name": "_get_available_memory", "qualname": "_get_available_memory", "full_name": "vllm_mlx.memory_cache._get_available_memory", "kind": "function", "signature": "def _get_available_memory() -> int", "parameters": [], "return_annotation": "int", "docstring": "Get available system memory in bytes.\n\nReturns:\n Available memory in bytes, or 0 if detection fails.", "summary": "Get available system memory in bytes.", "implementation": "Function `_get_available_memory` calls `psutil.virtual_memory`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 47, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L47-L63", "decorators": [], "calls": [ "psutil.virtual_memory", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "psutil.virtual_memory().available", "0" ], "awaits": false, "yields": false }, { "name": "_array_memory", "qualname": "_array_memory", "full_name": "vllm_mlx.memory_cache._array_memory", "kind": "function", "signature": "def _array_memory(arr) -> int", "parameters": [ { "name": "arr", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "An MLX array or similar object." } ], "return_annotation": "int", "docstring": "Estimate array memory from shape+dtype without triggering lazy eval.\n\nAccessing .nbytes on a lazy MLX array forces evaluation of the entire\ncomputation graph, causing a VRAM spike. This function uses shape and\ndtype metadata (which are always available without eval) to compute\nthe same value.\n\nArgs:\n arr: An MLX array or similar object.\n\nReturns:\n Estimated memory in bytes.", "summary": "Estimate array memory from shape+dtype without triggering lazy eval.", "implementation": "Function `_array_memory` calls `hasattr`, `math.prod`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 66, "end_line": 88, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L66-L88", "decorators": [], "calls": [ "hasattr", "math.prod" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "math.prod(arr.shape) * dtype.size", "arr.nbytes", "0" ], "awaits": false, "yields": false }, { "name": "_nested_array_memory", "qualname": "_nested_array_memory", "full_name": "vllm_mlx.memory_cache._nested_array_memory", "kind": "function", "signature": "def _nested_array_memory(value: Any) -> int", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Sum ``_array_memory`` over an arbitrarily nested state structure.\n\nCache ``state`` payloads are not always a flat ``(keys, values)`` pair:\nCacheList yields a list of sub-cache states and PoolingCache yields\n``(buf_kv, buf_gate, pooled)`` with possible ``None`` members. Unpacking\nthose as two values raised, was swallowed, and the entry was accounted as\nzero bytes — so the dashboard showed 0% cache memory and, far worse, the\nbyte-based LRU eviction never fired for such models.", "summary": "Sum ``_array_memory`` over an arbitrarily nested state structure.", "implementation": "Function `_nested_array_memory` calls `isinstance`, `sum`, `_nested_array_memory`, `_array_memory`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 91, "end_line": 105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L91-L105", "decorators": [], "calls": [ "isinstance", "sum", "_nested_array_memory", "_array_memory" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0", "sum((_nested_array_memory(v) for v in value))", "_array_memory(value)" ], "awaits": false, "yields": false }, { "name": "estimate_kv_cache_memory", "qualname": "estimate_kv_cache_memory", "full_name": "vllm_mlx.memory_cache.estimate_kv_cache_memory", "kind": "function", "signature": "def estimate_kv_cache_memory(cache: list[Any]) -> int", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "List of layer cache objects, each containing keys/values tensors." } ], "return_annotation": "int", "docstring": "Estimate memory usage of a KV cache in bytes.\n\nThis function inspects MLX arrays in the cache and calculates their\ntotal memory footprint using shape+dtype metadata to avoid triggering\nlazy evaluation (which would cause a VRAM spike).\n\nArgs:\n cache: List of layer cache objects, each containing keys/values tensors.\n\nReturns:\n Estimated memory usage in bytes.", "summary": "Estimate memory usage of a KV cache in bytes.", "implementation": "Function `estimate_kv_cache_memory` calls `isinstance`, `_array_memory`, `hasattr`, `getattr`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 108, "end_line": 162, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L108-L162", "decorators": [], "calls": [ "isinstance", "_array_memory", "hasattr", "getattr", "_nested_array_memory", "callable" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0", "total_bytes" ], "awaits": false, "yields": false }, { "name": "MemoryCacheConfig", "qualname": "MemoryCacheConfig", "full_name": "vllm_mlx.memory_cache.MemoryCacheConfig", "kind": "class", "signature": "class MemoryCacheConfig", "parameters": [ { "name": "max_memory_mb", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_memory_percent", "kind": "field", "annotation": "float", "default": "_DEFAULT_MEMORY_PERCENT", "required": false, "description": "Optional constructor field; defaults to `_DEFAULT_MEMORY_PERCENT`." }, { "name": "max_entries", "kind": "field", "annotation": "int", "default": "1000", "required": false, "description": "Optional constructor field; defaults to `1000`." }, { "name": "enable_memory_tracking", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "kv_quantize", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "kv_bits", "kind": "field", "annotation": "int", "default": "8", "required": false, "description": "Optional constructor field; defaults to `8`." }, { "name": "kv_group_size", "kind": "field", "annotation": "int", "default": "64", "required": false, "description": "Optional constructor field; defaults to `64`." }, { "name": "kv_min_quantize_tokens", "kind": "field", "annotation": "int", "default": "256", "required": false, "description": "Optional constructor field; defaults to `256`." }, { "name": "min_prefix_tokens", "kind": "field", "annotation": "int", "default": "128", "required": false, "description": "Optional constructor field; defaults to `128`." } ], "return_annotation": "MemoryCacheConfig", "docstring": "Configuration for memory-aware prefix cache.\n\nAttributes:\n max_memory_mb: Maximum memory in MB. If None, auto-detects.\n max_memory_percent: Fraction of available RAM to use (0.0-1.0).\n max_entries: Hard limit on number of entries (safety net).\n enable_memory_tracking: Whether to track per-entry memory.\n kv_quantize: Whether to quantize KV cache layers for reduced memory.\n kv_bits: Number of bits for KV cache quantization.\n kv_group_size: Group size for KV cache quantization.\n kv_min_quantize_tokens: Minimum sequence length for quantization to apply.\n min_prefix_tokens: Minimum cached prefix length eligible for reuse.", "summary": "Configuration for memory-aware prefix cache.", "implementation": "Class `MemoryCacheConfig` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 166, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L166-L225", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__post_init__", "qualname": "MemoryCacheConfig.__post_init__", "full_name": "vllm_mlx.memory_cache.MemoryCacheConfig.__post_init__", "kind": "method", "signature": "def __post_init__(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `MemoryCacheConfig.__post_init__` calls `ValueError`; can raise `ValueError`.", "implementation": "Method `MemoryCacheConfig.__post_init__` calls `ValueError`; can raise `ValueError`.", "documented": false, "public": false, "addressable": true, "line": 192, "end_line": 206, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L192-L206", "decorators": [], "calls": [ "ValueError" ], "state_reads": [ "self.max_memory_percent", "self.max_entries", "self.kv_min_quantize_tokens", "self.min_prefix_tokens" ], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "compute_memory_limit", "qualname": "MemoryCacheConfig.compute_memory_limit", "full_name": "vllm_mlx.memory_cache.MemoryCacheConfig.compute_memory_limit", "kind": "method", "signature": "def compute_memory_limit(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Compute the memory limit in bytes.\n\nReturns:\n Memory limit in bytes.", "summary": "Compute the memory limit in bytes.", "implementation": "Method `MemoryCacheConfig.compute_memory_limit` calls `_get_available_memory`, `int`, `max`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 208, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L208-L225", "decorators": [], "calls": [ "_get_available_memory", "int", "max" ], "state_reads": [ "self.max_memory_mb", "self.max_memory_percent" ], "state_writes": [], "raises": [], "return_expressions": [ "self.max_memory_mb * _BYTES_PER_MB", "max(limit, _MIN_MEMORY_BYTES)", "int(fallback_total * self.max_memory_percent)" ], "awaits": false, "yields": false }, { "name": "CacheStats", "qualname": "CacheStats", "full_name": "vllm_mlx.memory_cache.CacheStats", "kind": "class", "signature": "class CacheStats", "parameters": [ { "name": "hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "evictions", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "tokens_saved", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "current_memory_bytes", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "max_memory_bytes", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "entry_count", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "CacheStats", "docstring": "Statistics for cache performance monitoring.", "summary": "Statistics for cache performance monitoring.", "implementation": "Class `CacheStats` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 229, "end_line": 268, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L229-L268", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "hit_rate", "qualname": "CacheStats.hit_rate", "full_name": "vllm_mlx.memory_cache.CacheStats.hit_rate", "kind": "method", "signature": "def hit_rate(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Return successful lookups divided by all completed lookups.", "summary": "Return successful lookups divided by all completed lookups.", "implementation": "Method `CacheStats.hit_rate` returns `self.hits / total if total > 0 else 0.0`.", "documented": true, "public": true, "addressable": true, "line": 241, "end_line": 245, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L241-L245", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.hits", "self.misses" ], "state_writes": [], "raises": [], "return_expressions": [ "self.hits / total if total > 0 else 0.0" ], "awaits": false, "yields": false }, { "name": "memory_utilization", "qualname": "CacheStats.memory_utilization", "full_name": "vllm_mlx.memory_cache.CacheStats.memory_utilization", "kind": "method", "signature": "def memory_utilization(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Return the fraction of the configured memory budget in use.", "summary": "Return the fraction of the configured memory budget in use.", "implementation": "Method `CacheStats.memory_utilization` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 248, "end_line": 253, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L248-L253", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.max_memory_bytes", "self.current_memory_bytes" ], "state_writes": [], "raises": [], "return_expressions": [ "0.0", "self.current_memory_bytes / self.max_memory_bytes" ], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "CacheStats.to_dict", "full_name": "vllm_mlx.memory_cache.CacheStats.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict[str, Any]", "parameters": [], "return_annotation": "dict[str, Any]", "docstring": "Return rounded cache counters and memory values for APIs and logs.", "summary": "Return rounded cache counters and memory values for APIs and logs.", "implementation": "Method `CacheStats.to_dict` calls `round`; returns `{'hits': self.hits, 'misses': self.misses, 'hit_rate': round(self.hit_rate, 4), 'evictions': self.evictions, 'tokens_sa…`.", "documented": true, "public": true, "addressable": true, "line": 255, "end_line": 268, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L255-L268", "decorators": [], "calls": [ "round" ], "state_reads": [ "self.hits", "self.misses", "self.hit_rate", "self.evictions", "self.tokens_saved", "self.current_memory_bytes", "self.max_memory_bytes", "self.memory_utilization", "self.entry_count" ], "state_writes": [], "raises": [], "return_expressions": [ "{'hits': self.hits, 'misses': self.misses, 'hit_rate': round(self.hit_rate, 4), 'evictions': self.evictions, 'tokens_sa…" ], "awaits": false, "yields": false }, { "name": "_CacheEntry", "qualname": "_CacheEntry", "full_name": "vllm_mlx.memory_cache._CacheEntry", "kind": "class", "signature": "class _CacheEntry", "parameters": [ { "name": "tokens", "kind": "field", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "cache", "kind": "field", "annotation": "list[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "memory_bytes", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "_CacheEntry", "docstring": "Internal cache entry with memory tracking.", "summary": "Internal cache entry with memory tracking.", "implementation": "Class `_CacheEntry` declares 1 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 272, "end_line": 287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L272-L287", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "create", "qualname": "_CacheEntry.create", "full_name": "vllm_mlx.memory_cache._CacheEntry.create", "kind": "method", "signature": "def create(cls, tokens: list[int], cache: list[Any]) -> _CacheEntry", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_CacheEntry", "docstring": "Create a cache entry with memory estimation.", "summary": "Create a cache entry with memory estimation.", "implementation": "Method `_CacheEntry.create` calls `estimate_kv_cache_memory`, `cls`, `tuple`; returns `cls(tokens=tuple(tokens), cache=cache, memory_bytes=memory)`.", "documented": true, "public": false, "addressable": true, "line": 280, "end_line": 287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L280-L287", "decorators": [ "classmethod" ], "calls": [ "estimate_kv_cache_memory", "cls", "tuple" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cls(tokens=tuple(tokens), cache=cache, memory_bytes=memory)" ], "awaits": false, "yields": false }, { "name": "_is_cache_layer_trimmable", "qualname": "_is_cache_layer_trimmable", "full_name": "vllm_mlx.memory_cache._is_cache_layer_trimmable", "kind": "function", "signature": "def _is_cache_layer_trimmable(layer_cache: Any) -> bool", "parameters": [ { "name": "layer_cache", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return whether a cache layer can safely be rewound for partial reuse.", "summary": "Return whether a cache layer can safely be rewound for partial reuse.", "implementation": "Function `_is_cache_layer_trimmable` calls `isinstance`, `hasattr`, `getattr`, `callable`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 290, "end_line": 314, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L290-L314", "decorators": [], "calls": [ "isinstance", "hasattr", "getattr", "callable", "bool", "is_trimmable", "logger.debug", "type" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "hasattr(layer_cache, 'offset') and hasattr(layer_cache, 'keys')", "bool(is_trimmable())" ], "awaits": false, "yields": false }, { "name": "_trim_cache_offset", "qualname": "_trim_cache_offset", "full_name": "vllm_mlx.memory_cache._trim_cache_offset", "kind": "function", "signature": "def _trim_cache_offset(cache: list[Any], trim_by: int) -> list[Any]", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "trim_by", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[Any]", "docstring": "Create copies of cache layers with the last ``trim_by`` positions removed.\n\nThis is used when returning a cached KV state to the scheduler so that\nthe last N positions are \"freed\" and the model will recompute them on the\nnext forward pass (preventing duplicate KV entries).\n\nFor plain KVCache: reduces offset (surplus data beyond offset is harmless\nsince merge slices to ``keys[:, :, :offset, :]``).\n\nFor RotatingKVCache: actually trims the circular buffer — reducing offset\nalone breaks ``size()`` / ``_temporal_order`` invariants.\n\nSupports KVCache, RotatingKVCache, and _QuantizedCacheWrapper.", "summary": "Create copies of cache layers with the last ``trim_by`` positions removed.", "implementation": "Function `_trim_cache_offset` calls `isinstance`, `_QuantizedCacheWrapper.__new__`, `max`, `trimmed.append`; returns `trimmed`.", "documented": true, "public": false, "addressable": true, "line": 317, "end_line": 481, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L317-L481", "decorators": [], "calls": [ "isinstance", "_QuantizedCacheWrapper.__new__", "max", "trimmed.append", "min", "type", "orig_cls.__new__", "getattr", "layer_cache._temporal_order", "mx.zeros", "mx.concatenate", "eval_targets.extend", "hasattr", "len", "setattr", "mx.eval" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "trimmed" ], "awaits": false, "yields": false }, { "name": "_needs_kv_trim", "qualname": "_needs_kv_trim", "full_name": "vllm_mlx.memory_cache._needs_kv_trim", "kind": "function", "signature": "def _needs_kv_trim(layer: Any) -> bool", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if a cache layer has oversized KV arrays (duck-typed, no MLX import).", "summary": "Check if a cache layer has oversized KV arrays (duck-typed, no MLX import).", "implementation": "Function `_needs_kv_trim` calls `getattr`, `isinstance`, `len`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 484, "end_line": 495, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L484-L495", "decorators": [], "calls": [ "getattr", "isinstance", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "0 < offset < shape[2]" ], "awaits": false, "yields": false }, { "name": "_trim_to_offset", "qualname": "_trim_to_offset", "full_name": "vllm_mlx.memory_cache._trim_to_offset", "kind": "function", "signature": "def _trim_to_offset(cache: list[Any]) -> list[Any]", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "List of cache layer objects (KVCache or other types)." } ], "return_annotation": "list[Any]", "docstring": "Trim KV arrays to their actual used size (offset) before storage.\n\nKV arrays are often pre-allocated larger than needed (e.g. 4096 slots\nwhen only 100 are used). This slices them down to ``offset`` and\nevaluates the result so the original large buffer can be freed.\n\nArgs:\n cache: List of cache layer objects (KVCache or other types).\n\nReturns:\n New list with KVCache layers trimmed to their offset.\n Non-KVCache layers are passed through unchanged.", "summary": "Trim KV arrays to their actual used size (offset) before storage.", "implementation": "Function `_trim_to_offset` calls `any`, `_needs_kv_trim`, `isinstance`, `trimmed.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 498, "end_line": 538, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L498-L538", "decorators": [], "calls": [ "any", "_needs_kv_trim", "isinstance", "trimmed.append", "KVCache", "eval_targets.extend", "mx.eval" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cache", "trimmed" ], "awaits": false, "yields": false }, { "name": "_QuantizedCacheWrapper", "qualname": "_QuantizedCacheWrapper", "full_name": "vllm_mlx.memory_cache._QuantizedCacheWrapper", "kind": "class", "signature": "class _QuantizedCacheWrapper", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "bits", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "group_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_QuantizedCacheWrapper", "docstring": "Lightweight wrapper storing quantized KV arrays + original cache metadata.\n\nUnlike ``QuantizedKVCache``, this preserves enough info to reconstruct\nthe *original* cache type (KVCache, RotatingKVCache, etc.) on dequantize.", "summary": "Lightweight wrapper storing quantized KV arrays + original cache metadata.", "implementation": "Class `_QuantizedCacheWrapper` declares 1 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 541, "end_line": 571, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L541-L571", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_QuantizedCacheWrapper.__init__", "full_name": "vllm_mlx.memory_cache._QuantizedCacheWrapper.__init__", "kind": "method", "signature": "def __init__(self, layer: Any, bits: int, group_size: int)", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "bits", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "group_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_QuantizedCacheWrapper.__init__` updates `self.keys`, `self.values`, `self.offset`, `self.bits`; calls `mx.quantize`, `type`, `hasattr`, `getattr`.", "implementation": "Method `_QuantizedCacheWrapper.__init__` updates `self.keys`, `self.values`, `self.offset`, `self.bits`; calls `mx.quantize`, `type`, `hasattr`, `getattr`.", "documented": false, "public": false, "addressable": true, "line": 558, "end_line": 571, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L558-L571", "decorators": [], "calls": [ "mx.quantize", "type", "hasattr", "getattr" ], "state_reads": [ "self.orig_attrs" ], "state_writes": [ "self.keys", "self.values", "self.offset", "self.bits", "self.group_size", "self.orig_type", "self.orig_attrs" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_quantize_cache", "qualname": "_quantize_cache", "full_name": "vllm_mlx.memory_cache._quantize_cache", "kind": "function", "signature": "def _quantize_cache(cache: list[Any], bits: int=8, group_size: int=64) -> list[Any]", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "bits", "kind": "positional or keyword", "annotation": "int", "default": "8", "required": false, "description": "Optional positional or keyword input; defaults to `8`." }, { "name": "group_size", "kind": "positional or keyword", "annotation": "int", "default": "64", "required": false, "description": "Optional positional or keyword input; defaults to `64`." } ], "return_annotation": "list[Any]", "docstring": "Quantize KV cache layers to reduce memory.\n\nOnly plain KVCache layers are quantized. RotatingKVCache (sliding window)\nis left as-is because its internal _idx/rotation state is tightly coupled\nwith update_and_fetch logic and cannot survive quantize/dequantize roundtrip.\nRotatingKVCache is typically small (max_size=1024) so skipping it is fine.", "summary": "Quantize KV cache layers to reduce memory.", "implementation": "Function `_quantize_cache` calls `type`, `getattr`, `quantized.append`, `_QuantizedCacheWrapper`; returns `quantized`.", "documented": true, "public": false, "addressable": true, "line": 574, "end_line": 590, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L574-L590", "decorators": [], "calls": [ "type", "getattr", "quantized.append", "_QuantizedCacheWrapper" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "quantized" ], "awaits": false, "yields": false }, { "name": "_dequantize_cache", "qualname": "_dequantize_cache", "full_name": "vllm_mlx.memory_cache._dequantize_cache", "kind": "function", "signature": "def _dequantize_cache(cache: list[Any]) -> list[Any]", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[Any]", "docstring": "Dequantize _QuantizedCacheWrapper layers and copy non-quantized layers.\n\nAll layers are copied (never returned by reference) so that the model's\n``update_and_fetch`` mutations don't corrupt the stored cache entry.", "summary": "Dequantize _QuantizedCacheWrapper layers and copy non-quantized layers.", "implementation": "Function `_dequantize_cache` calls `isinstance`, `orig_cls.__new__`, `mx.dequantize`, `hasattr`; returns `result`.", "documented": true, "public": false, "addressable": true, "line": 593, "end_line": 645, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L593-L645", "decorators": [], "calls": [ "isinstance", "orig_cls.__new__", "mx.dequantize", "hasattr", "len", "layer.orig_attrs.items", "setattr", "result.append", "type", "mx.array", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "_compute_model_fingerprint", "qualname": "_compute_model_fingerprint", "full_name": "vllm_mlx.memory_cache._compute_model_fingerprint", "kind": "function", "signature": "def _compute_model_fingerprint(model: Any) -> str", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Compute a fingerprint from model architecture for cache compatibility.\n\nUsed to reject disk-persisted caches created by a different model or\na different quantisation of the same model. The fingerprint is a\nshort hex digest of (num_layers, hidden_size, vocab_size, num_kv_heads,\nhead_dim) — lightweight and deterministic.", "summary": "Compute a fingerprint from model architecture for cache compatibility.", "implementation": "Function `_compute_model_fingerprint` calls `getattr`, `parts.append`, `hashlib.sha256('|'.join(parts).encode()).hexdigest`, `hashlib.sha256`; returns `fingerprint`.", "documented": true, "public": false, "addressable": true, "line": 648, "end_line": 682, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L648-L682", "decorators": [], "calls": [ "getattr", "parts.append", "hashlib.sha256('|'.join(parts).encode()).hexdigest", "hashlib.sha256", "'|'.join(parts).encode", "'|'.join", "logger.debug", "', '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "fingerprint" ], "awaits": false, "yields": false }, { "name": "MemoryAwarePrefixCache", "qualname": "MemoryAwarePrefixCache", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache", "kind": "class", "signature": "class MemoryAwarePrefixCache", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model (used for identification)." }, { "name": "config", "kind": "positional or keyword", "annotation": "MemoryCacheConfig | None", "default": "None", "required": false, "description": "Cache configuration. Uses defaults if None." } ], "return_annotation": "MemoryAwarePrefixCache", "docstring": "Prefix cache with memory-based eviction.\n\nThis cache tracks memory usage per entry and evicts based on memory\npressure rather than entry count. It uses LRU (Least Recently Used)\nordering for eviction decisions.\n\nKey design decisions:\n- No deep copies on fetch: MLX arrays are immutable, so sharing is safe\n- Memory tracking per entry: Accurate accounting for eviction\n- Auto-detection of available RAM: Adapts to different systems\n- OrderedDict for O(1) LRU operations\n\nThread Safety:\n This class is NOT thread-safe. Use external locking if needed.", "summary": "Prefix cache with memory-based eviction.", "implementation": "Class `MemoryAwarePrefixCache` declares 19 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 685, "end_line": 1463, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L685-L1463", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MemoryAwarePrefixCache.__init__", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.__init__", "kind": "method", "signature": "def __init__(self, model: Any, config: MemoryCacheConfig | None=None) -> None", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model (used for identification)." }, { "name": "config", "kind": "positional or keyword", "annotation": "MemoryCacheConfig | None", "default": "None", "required": false, "description": "Cache configuration. Uses defaults if None." } ], "return_annotation": "None", "docstring": "Initialize the memory-aware prefix cache.\n\nArgs:\n model: The MLX model (used for identification).\n config: Cache configuration. Uses defaults if None.", "summary": "Initialize the memory-aware prefix cache.", "implementation": "Method `MemoryAwarePrefixCache.__init__` updates `self._model_id`, `self._config`, `self._model_fingerprint`, `self._entries`; calls `id`, `MemoryCacheConfig`, `_compute_model_fingerprint`, `OrderedDict`.", "documented": true, "public": false, "addressable": true, "line": 703, "end_line": 746, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L703-L746", "decorators": [], "calls": [ "id", "MemoryCacheConfig", "_compute_model_fingerprint", "OrderedDict", "self._config.compute_memory_limit", "threading.RLock", "CacheStats", "logger.info" ], "state_reads": [ "self._config.compute_memory_limit", "self._config", "self._max_memory", "self._config.max_entries" ], "state_writes": [ "self._model_id", "self._config", "self._model_fingerprint", "self._entries", "self._sorted_keys", "self._max_memory", "self._current_memory", "self._memory_lock", "self._stats", "self._last_match_type", "self._ssd_tier" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "fetch", "qualname": "MemoryAwarePrefixCache.fetch", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.fetch", "kind": "method", "signature": "def fetch(self, tokens: list[int]) -> tuple[list[Any] | None, list[int]]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Input token sequence." } ], "return_annotation": "tuple[list[Any] | None, list[int]]", "docstring": "Find cached KV state for the given tokens.\n\nThis method searches for exact matches, prefix matches, supersequence\nmatches, and longest-common-prefix (LCP) matches. Uses a sorted key\nindex for O(log N) lookup instead of scanning all entries.\n\nReturns the cached KV state directly (no copy) since MLX arrays\nare immutable and safe to share.\n\nArgs:\n tokens: Input token sequence.\n\nReturns:\n Tuple of (cache, remaining_tokens):\n - cache: Cached KV state if found, None otherwise\n - remaining_tokens: Tokens that still need processing", "summary": "Find cached KV state for the given tokens.", "implementation": "Method `MemoryAwarePrefixCache.fetch` updates `self._stats.misses`, `self._last_match_type`, `self._stats.hits`, `self._stats.tokens_saved`; calls `len`, `tuple`, `self._entries.move_to_end`, `_dequantize_cache`; has 5 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 748, "end_line": 977, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L748-L977", "decorators": [], "calls": [ "len", "tuple", "self._entries.move_to_end", "_dequantize_cache", "bisect.bisect_left", "range", "any", "_is_cache_layer_trimmable", "logger.debug", "_trim_cache_offset", "min", "type" ], "state_reads": [ "self._stats", "self._config.min_prefix_tokens", "self._config", "self._entries", "self._entries.move_to_end", "self._config.kv_quantize", "self._sorted_keys" ], "state_writes": [ "self._stats.misses", "self._last_match_type", "self._stats.hits", "self._stats.tokens_saved" ], "raises": [], "return_expressions": [ "(None, tokens)", "(cache_out, [])", "(trimmed_cache, [])", "(cache_out, remaining)", "(trimmed_cache, remaining)" ], "awaits": false, "yields": false }, { "name": "store", "qualname": "MemoryAwarePrefixCache.store", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.store", "kind": "method", "signature": "def store(self, tokens: list[int], cache: list[Any], evict_prefixes: bool=True) -> bool", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Token sequence that was processed." }, { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "The computed KV cache to store." }, { "name": "evict_prefixes", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, evict existing entries whose token sequence is a strict prefix of ``tokens``. Set to False when storing prompt+output entries to preserve prompt-only entries created by prompt_cache_save (those are the entries that future requests will actually match)." } ], "return_annotation": "bool", "docstring": "Store KV cache for future reuse.\n\nThis method stores the cache reference directly (no copy) and\ntracks memory usage. If memory limit is exceeded, LRU entries\nare evicted until there's room.\n\nArgs:\n tokens: Token sequence that was processed.\n cache: The computed KV cache to store.\n evict_prefixes: If True, evict existing entries whose token\n sequence is a strict prefix of ``tokens``. Set to False\n when storing prompt+output entries to preserve prompt-only\n entries created by prompt_cache_save (those are the entries\n that future requests will actually match).\n\nReturns:\n True if stored successfully, False if rejected.", "summary": "Store KV cache for future reuse.", "implementation": "Method `MemoryAwarePrefixCache.store` updates `self._current_memory`, `self._stats.evictions`, `self._stats.entry_count`, `self._stats.current_memory_bytes`; calls `len`, `logger.debug`, `tuple`, `self._entries.move_to_end`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 979, "end_line": 1092, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L979-L1092", "decorators": [], "calls": [ "len", "logger.debug", "tuple", "self._entries.move_to_end", "_trim_to_offset", "_quantize_cache", "_CacheEntry.create", "logger.warning", "bisect.bisect_left", "range", "to_remove.append", "self._entries.pop", "self._remove_from_sorted", "self._evict_lru", "bisect.insort" ], "state_reads": [ "self._config.min_prefix_tokens", "self._config", "self._memory_lock", "self._entries", "self._entries.move_to_end", "self._config.kv_quantize", "self._config.kv_min_quantize_tokens", "self._config.kv_bits", "self._config.kv_group_size", "self._max_memory", "self._sorted_keys", "self._entries.pop", "self._stats", "self._remove_from_sorted", "self._current_memory", "self._config.max_entries", "self._evict_lru" ], "state_writes": [ "self._current_memory", "self._stats.evictions", "self._stats.entry_count", "self._stats.current_memory_bytes" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_remove_from_sorted", "qualname": "MemoryAwarePrefixCache._remove_from_sorted", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache._remove_from_sorted", "kind": "method", "signature": "def _remove_from_sorted(self, key: tuple[int, ...]) -> None", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Remove a key from the sorted index using bisect for O(log N).", "summary": "Remove a key from the sorted index using bisect for O(log N).", "implementation": "Method `MemoryAwarePrefixCache._remove_from_sorted` calls `bisect.bisect_left`, `len`, `self._sorted_keys.pop`.", "documented": true, "public": false, "addressable": true, "line": 1094, "end_line": 1098, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1094-L1098", "decorators": [], "calls": [ "bisect.bisect_left", "len", "self._sorted_keys.pop" ], "state_reads": [ "self._sorted_keys", "self._sorted_keys.pop" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_evict_lru", "qualname": "MemoryAwarePrefixCache._evict_lru", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache._evict_lru", "kind": "method", "signature": "def _evict_lru(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Evict the least recently used entry.\n\nIf an SSD tier is attached, the entry is spilled to disk instead\nof being discarded.", "summary": "Evict the least recently used entry.", "implementation": "Method `MemoryAwarePrefixCache._evict_lru` updates `self._current_memory`, `self._stats.evictions`, `self._stats.entry_count`, `self._stats.current_memory_bytes`; calls `self._entries.popitem`, `self._remove_from_sorted`, `len`, `self._ssd_tier.enqueue_spill`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1100, "end_line": 1126, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1100-L1126", "decorators": [], "calls": [ "self._entries.popitem", "self._remove_from_sorted", "len", "self._ssd_tier.enqueue_spill", "logger.debug" ], "state_reads": [ "self._memory_lock", "self._entries", "self._entries.popitem", "self._remove_from_sorted", "self._stats", "self._current_memory", "self._ssd_tier", "self._ssd_tier.enqueue_spill" ], "state_writes": [ "self._current_memory", "self._stats.evictions", "self._stats.entry_count", "self._stats.current_memory_bytes" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "remove", "qualname": "MemoryAwarePrefixCache.remove", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.remove", "kind": "method", "signature": "def remove(self, tokens: list[int]) -> bool", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Token sequence to remove." } ], "return_annotation": "bool", "docstring": "Remove a specific cache entry.\n\nArgs:\n tokens: Token sequence to remove.\n\nReturns:\n True if entry was found and removed.", "summary": "Remove a specific cache entry.", "implementation": "Method `MemoryAwarePrefixCache.remove` updates `self._current_memory`, `self._stats.entry_count`, `self._stats.current_memory_bytes`; calls `tuple`, `self._entries.pop`, `self._remove_from_sorted`, `len`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1128, "end_line": 1147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1128-L1147", "decorators": [], "calls": [ "tuple", "self._entries.pop", "self._remove_from_sorted", "len" ], "state_reads": [ "self._memory_lock", "self._entries.pop", "self._entries", "self._remove_from_sorted", "self._stats", "self._current_memory" ], "state_writes": [ "self._current_memory", "self._stats.entry_count", "self._stats.current_memory_bytes" ], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "clear", "qualname": "MemoryAwarePrefixCache.clear", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear all cached entries.", "summary": "Clear all cached entries.", "implementation": "Method `MemoryAwarePrefixCache.clear` updates `self._current_memory`, `self._stats`; calls `self._entries.clear`, `self._sorted_keys.clear`, `CacheStats`, `logger.debug`.", "documented": true, "public": true, "addressable": true, "line": 1149, "end_line": 1156, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1149-L1156", "decorators": [], "calls": [ "self._entries.clear", "self._sorted_keys.clear", "CacheStats", "logger.debug" ], "state_reads": [ "self._memory_lock", "self._entries.clear", "self._entries", "self._sorted_keys.clear", "self._sorted_keys", "self._max_memory" ], "state_writes": [ "self._current_memory", "self._stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "MemoryAwarePrefixCache.get_stats", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict[str, Any]", "parameters": [], "return_annotation": "dict[str, Any]", "docstring": "Get cache statistics.", "summary": "Get cache statistics.", "implementation": "Method `MemoryAwarePrefixCache.get_stats` calls `self._stats.to_dict`; returns `self._stats.to_dict()`.", "documented": true, "public": true, "addressable": true, "line": 1158, "end_line": 1160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1158-L1160", "decorators": [], "calls": [ "self._stats.to_dict" ], "state_reads": [ "self._stats.to_dict", "self._stats" ], "state_writes": [], "raises": [], "return_expressions": [ "self._stats.to_dict()" ], "awaits": false, "yields": false }, { "name": "reset_stats", "qualname": "MemoryAwarePrefixCache.reset_stats", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.reset_stats", "kind": "method", "signature": "def reset_stats(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset statistics while preserving cache contents.", "summary": "Reset statistics while preserving cache contents.", "implementation": "Method `MemoryAwarePrefixCache.reset_stats` updates `self._stats`; calls `CacheStats`, `len`.", "documented": true, "public": true, "addressable": true, "line": 1162, "end_line": 1169, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1162-L1169", "decorators": [], "calls": [ "CacheStats", "len" ], "state_reads": [ "self._memory_lock", "self._max_memory", "self._current_memory", "self._entries" ], "state_writes": [ "self._stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "memory_usage_mb", "qualname": "MemoryAwarePrefixCache.memory_usage_mb", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.memory_usage_mb", "kind": "method", "signature": "def memory_usage_mb(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Current memory usage in MB.", "summary": "Current memory usage in MB.", "implementation": "Method `MemoryAwarePrefixCache.memory_usage_mb` returns `self._current_memory / _BYTES_PER_MB`.", "documented": true, "public": true, "addressable": true, "line": 1172, "end_line": 1174, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1172-L1174", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._current_memory" ], "state_writes": [], "raises": [], "return_expressions": [ "self._current_memory / _BYTES_PER_MB" ], "awaits": false, "yields": false }, { "name": "memory_limit_mb", "qualname": "MemoryAwarePrefixCache.memory_limit_mb", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.memory_limit_mb", "kind": "method", "signature": "def memory_limit_mb(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Memory limit in MB.", "summary": "Memory limit in MB.", "implementation": "Method `MemoryAwarePrefixCache.memory_limit_mb` returns `self._max_memory / _BYTES_PER_MB`.", "documented": true, "public": true, "addressable": true, "line": 1177, "end_line": 1179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1177-L1179", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._max_memory" ], "state_writes": [], "raises": [], "return_expressions": [ "self._max_memory / _BYTES_PER_MB" ], "awaits": false, "yields": false }, { "name": "try_reserve_memory", "qualname": "MemoryAwarePrefixCache.try_reserve_memory", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.try_reserve_memory", "kind": "method", "signature": "def try_reserve_memory(self, nbytes: int) -> bool", "parameters": [ { "name": "nbytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Tentatively reserve cache memory for an upcoming promotion.", "summary": "Tentatively reserve cache memory for an upcoming promotion.", "implementation": "Method `MemoryAwarePrefixCache.try_reserve_memory` updates `self._current_memory`, `self._stats.current_memory_bytes`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1181, "end_line": 1188, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1181-L1188", "decorators": [], "calls": [], "state_reads": [ "self._memory_lock", "self._current_memory", "self._max_memory", "self._stats" ], "state_writes": [ "self._current_memory", "self._stats.current_memory_bytes" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "release_reserved_memory", "qualname": "MemoryAwarePrefixCache.release_reserved_memory", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.release_reserved_memory", "kind": "method", "signature": "def release_reserved_memory(self, nbytes: int) -> None", "parameters": [ { "name": "nbytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Release memory previously reserved by try_reserve_memory().", "summary": "Release memory previously reserved by try_reserve_memory().", "implementation": "Method `MemoryAwarePrefixCache.release_reserved_memory` updates `self._current_memory`, `self._stats.current_memory_bytes`; calls `max`.", "documented": true, "public": true, "addressable": true, "line": 1190, "end_line": 1194, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1190-L1194", "decorators": [], "calls": [ "max" ], "state_reads": [ "self._memory_lock", "self._current_memory", "self._stats" ], "state_writes": [ "self._current_memory", "self._stats.current_memory_bytes" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "MemoryAwarePrefixCache.__len__", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.__len__", "kind": "method", "signature": "def __len__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return number of cached entries.", "summary": "Return number of cached entries.", "implementation": "Method `MemoryAwarePrefixCache.__len__` calls `len`; returns `len(self._entries)`.", "documented": true, "public": false, "addressable": true, "line": 1196, "end_line": 1198, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1196-L1198", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._entries" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self._entries)" ], "awaits": false, "yields": false }, { "name": "__contains__", "qualname": "MemoryAwarePrefixCache.__contains__", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.__contains__", "kind": "method", "signature": "def __contains__(self, tokens: list[int]) -> bool", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if tokens are cached.", "summary": "Check if tokens are cached.", "implementation": "Method `MemoryAwarePrefixCache.__contains__` calls `tuple`; returns `tuple(tokens) in self._entries`.", "documented": true, "public": false, "addressable": true, "line": 1200, "end_line": 1202, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1200-L1202", "decorators": [], "calls": [ "tuple" ], "state_reads": [ "self._entries" ], "state_writes": [], "raises": [], "return_expressions": [ "tuple(tokens) in self._entries" ], "awaits": false, "yields": false }, { "name": "set_ssd_tier", "qualname": "MemoryAwarePrefixCache.set_ssd_tier", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.set_ssd_tier", "kind": "method", "signature": "def set_ssd_tier(self, ssd_tier) -> None", "parameters": [ { "name": "ssd_tier", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "An SSDCacheTier instance (or None to disable)." } ], "return_annotation": "None", "docstring": "Attach an SSD cache tier for eviction spilling.\n\nWhen set, evicted entries are spilled to SSD instead of discarded.\n\nArgs:\n ssd_tier: An SSDCacheTier instance (or None to disable).", "summary": "Attach an SSD cache tier for eviction spilling.", "implementation": "Method `MemoryAwarePrefixCache.set_ssd_tier` updates `self._ssd_tier`; calls `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 1204, "end_line": 1214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1204-L1214", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [ "self._ssd_tier" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "check_ssd", "qualname": "MemoryAwarePrefixCache.check_ssd", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.check_ssd", "kind": "method", "signature": "def check_ssd(self, tokens: list[int]) -> dict | None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict | None", "docstring": "Check if tokens have an SSD cache hit (without reading data).\n\nReturns metadata dict with 'match_type' ('exact' or 'prefix') if\nfound in SSD tier, None if not found. For prefix matches, the dict\nalso includes 'matched_tokens' (the count of tokens the SSD entry\ncovers).\n\nThis is a fast synchronous call (SQLite lookup only).\nThe actual data read happens via the scheduler handoff.", "summary": "Check if tokens have an SSD cache hit (without reading data).", "implementation": "Method `MemoryAwarePrefixCache.check_ssd` calls `tuple`, `self._ssd_tier.lookup_ssd`, `len`, `self._ssd_tier.lookup_ssd_prefix`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1216, "end_line": 1249, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1216-L1249", "decorators": [], "calls": [ "tuple", "self._ssd_tier.lookup_ssd", "len", "self._ssd_tier.lookup_ssd_prefix" ], "state_reads": [ "self._ssd_tier", "self._entries", "self._ssd_tier.lookup_ssd", "self._ssd_tier.lookup_ssd_prefix" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "candidate", "prefix" ], "awaits": false, "yields": false }, { "name": "save_to_disk", "qualname": "MemoryAwarePrefixCache.save_to_disk", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.save_to_disk", "kind": "method", "signature": "def save_to_disk(self, cache_dir: str) -> bool", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Save all cache entries to disk using mlx_lm's safetensors format.\n\nDirectory layout::\n\n cache_dir/\n index.json # token keys + metadata per entry\n entry_0.safetensors # KV arrays for entry 0\n entry_1.safetensors\n ...\n\nReturns True if at least one entry was saved.", "summary": "Save all cache entries to disk using mlx_lm's safetensors format.", "implementation": "Method `MemoryAwarePrefixCache.save_to_disk` calls `logger.info`, `_time.monotonic`, `os.makedirs`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1255, "end_line": 1346, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1255-L1346", "decorators": [], "calls": [ "logger.info", "_time.monotonic", "os.makedirs", "logger.warning", "len", "enumerate", "self._entries.items", "os.path.join", "any", "isinstance", "_dequantize_cache", "save_prompt_cache", "str", "_array.array", "open", "arr.tofile", "index['entries'].append", "json.dump" ], "state_reads": [ "self._entries", "self._model_fingerprint", "self._current_memory", "self._entries.items" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "saved > 0" ], "awaits": false, "yields": false }, { "name": "load_from_disk", "qualname": "MemoryAwarePrefixCache.load_from_disk", "full_name": "vllm_mlx.memory_cache.MemoryAwarePrefixCache.load_from_disk", "kind": "method", "signature": "def load_from_disk(self, cache_dir: str) -> int", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Load cache entries from disk.\n\nReturns the number of entries successfully loaded.", "summary": "Load cache entries from disk.", "implementation": "Method `MemoryAwarePrefixCache.load_from_disk` updates `self._current_memory`, `self._stats.entry_count`, `self._stats.current_memory_bytes`; calls `os.path.join`, `os.path.exists`, `logger.info`, `_time.monotonic`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1348, "end_line": 1463, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/memory_cache.py#L1348-L1463", "decorators": [], "calls": [ "os.path.join", "os.path.exists", "logger.info", "_time.monotonic", "logger.warning", "open", "json.load", "index.get", "_array.array", "arr.fromfile", "list", "len", "load_prompt_cache", "estimate_kv_cache_memory", "tuple", "_CacheEntry", "bisect.insort" ], "state_reads": [ "self._model_fingerprint", "self._config.min_prefix_tokens", "self._config", "self._memory_lock", "self._current_memory", "self._max_memory", "self._entries", "self._sorted_keys", "self._stats" ], "state_writes": [ "self._current_memory", "self._stats.entry_count", "self._stats.current_memory_bytes" ], "raises": [], "return_expressions": [ "0", "loaded" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.metrics", "path": "vllm_mlx/metrics.py", "page_path": "reference/api/vllm_mlx/metrics.md", "docstring": "Prometheus-first server metrics for vllm-mlx.\n\nThe public surface is a small internal abstraction that keeps instrumentation\ncall sites stable even if we add OpenTelemetry export later.", "summary": "Prometheus-first server metrics for vllm-mlx.", "line_count": 532, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L1-L532", "members": [ "_bool_str", "_coerce_float", "_coerce_int", "InferenceTracker", "MetricsCollector", "metrics" ], "symbols": [ { "name": "_bool_str", "qualname": "_bool_str", "full_name": "vllm_mlx.metrics._bool_str", "kind": "function", "signature": "def _bool_str(value: bool) -> str", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Function `_bool_str` returns `'true' if value else 'false'`.", "implementation": "Function `_bool_str` returns `'true' if value else 'false'`.", "documented": false, "public": false, "addressable": true, "line": 17, "end_line": 18, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L17-L18", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'true' if value else 'false'" ], "awaits": false, "yields": false }, { "name": "_coerce_float", "qualname": "_coerce_float", "full_name": "vllm_mlx.metrics._coerce_float", "kind": "function", "signature": "def _coerce_float(value: Any, default: float=0.0) -> float", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "default", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Optional positional or keyword input; defaults to `0.0`." } ], "return_annotation": "float", "docstring": "", "summary": "Function `_coerce_float` calls `float`; has 2 explicit return paths.", "implementation": "Function `_coerce_float` calls `float`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 21, "end_line": 27, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L21-L27", "decorators": [], "calls": [ "float" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "default", "float(value)" ], "awaits": false, "yields": false }, { "name": "_coerce_int", "qualname": "_coerce_int", "full_name": "vllm_mlx.metrics._coerce_int", "kind": "function", "signature": "def _coerce_int(value: Any, default: int=0) -> int", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "default", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "int", "docstring": "", "summary": "Function `_coerce_int` calls `int`; has 2 explicit return paths.", "implementation": "Function `_coerce_int` calls `int`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 30, "end_line": 36, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L30-L36", "decorators": [], "calls": [ "int" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "default", "int(value)" ], "awaits": false, "yields": false }, { "name": "InferenceTracker", "qualname": "InferenceTracker", "full_name": "vllm_mlx.metrics.InferenceTracker", "kind": "class", "signature": "class InferenceTracker", "parameters": [ { "name": "collector", "kind": "field", "annotation": "'MetricsCollector | None'", "default": "", "required": true, "description": "Required constructor field." }, { "name": "endpoint", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "stream", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "start_time", "kind": "field", "annotation": "float", "default": "field(default_factory=time.perf_counter)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=time.perf_counter)`." }, { "name": "_finished", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "_ttft_observed", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "InferenceTracker", "docstring": "Request-scoped inference timing and token accounting.", "summary": "Request-scoped inference timing and token accounting.", "implementation": "Class `InferenceTracker` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 81, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L40-L81", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "observe_ttft", "qualname": "InferenceTracker.observe_ttft", "full_name": "vllm_mlx.metrics.InferenceTracker.observe_ttft", "kind": "method", "signature": "def observe_ttft(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Record time to first token once for this inference request.", "summary": "Record time to first token once for this inference request.", "implementation": "Method `InferenceTracker.observe_ttft` updates `self._ttft_observed`; calls `self.collector.observe_ttft`, `time.perf_counter`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 50, "end_line": 60, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L50-L60", "decorators": [], "calls": [ "self.collector.observe_ttft", "time.perf_counter" ], "state_reads": [ "self.collector", "self._ttft_observed", "self.collector.observe_ttft", "self.endpoint", "self.stream", "self.start_time" ], "state_writes": [ "self._ttft_observed" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "finish", "qualname": "InferenceTracker.finish", "full_name": "vllm_mlx.metrics.InferenceTracker.finish", "kind": "method", "signature": "def finish(self, *, result: str, prompt_tokens: int=0, completion_tokens: int=0) -> None", "parameters": [ { "name": "result", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "prompt_tokens", "kind": "keyword-only", "annotation": "int", "default": "0", "required": false, "description": "Optional keyword-only input; defaults to `0`." }, { "name": "completion_tokens", "kind": "keyword-only", "annotation": "int", "default": "0", "required": false, "description": "Optional keyword-only input; defaults to `0`." } ], "return_annotation": "None", "docstring": "Record terminal latency and token counts once for this request.", "summary": "Record terminal latency and token counts once for this request.", "implementation": "Method `InferenceTracker.finish` updates `self._finished`; calls `self.collector.observe_inference`, `time.perf_counter`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 62, "end_line": 81, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L62-L81", "decorators": [], "calls": [ "self.collector.observe_inference", "time.perf_counter" ], "state_reads": [ "self.collector", "self._finished", "self.collector.observe_inference", "self.endpoint", "self.stream", "self.start_time" ], "state_writes": [ "self._finished" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "MetricsCollector", "qualname": "MetricsCollector", "full_name": "vllm_mlx.metrics.MetricsCollector", "kind": "class", "signature": "class MetricsCollector", "parameters": [], "return_annotation": "MetricsCollector", "docstring": "Lazy Prometheus-backed metrics collector.", "summary": "Lazy Prometheus-backed metrics collector.", "implementation": "Class `MetricsCollector` declares 11 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 84, "end_line": 529, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L84-L529", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MetricsCollector.__init__", "full_name": "vllm_mlx.metrics.MetricsCollector.__init__", "kind": "method", "signature": "def __init__(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `MetricsCollector.__init__` updates `self._enabled`, `self._lock`, `self._prom`; calls `threading.Lock`.", "implementation": "Method `MetricsCollector.__init__` updates `self._enabled`, `self._lock`, `self._prom`; calls `threading.Lock`.", "documented": false, "public": false, "addressable": true, "line": 87, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L87-L90", "decorators": [], "calls": [ "threading.Lock" ], "state_reads": [], "state_writes": [ "self._enabled", "self._lock", "self._prom" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "enabled", "qualname": "MetricsCollector.enabled", "full_name": "vllm_mlx.metrics.MetricsCollector.enabled", "kind": "method", "signature": "def enabled(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return whether metric collection is enabled.", "summary": "Return whether metric collection is enabled.", "implementation": "Method `MetricsCollector.enabled` returns `self._enabled`.", "documented": true, "public": true, "addressable": true, "line": 93, "end_line": 96, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L93-L96", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._enabled" ], "state_writes": [], "raises": [], "return_expressions": [ "self._enabled" ], "awaits": false, "yields": false }, { "name": "configure", "qualname": "MetricsCollector.configure", "full_name": "vllm_mlx.metrics.MetricsCollector.configure", "kind": "method", "signature": "def configure(self, *, enabled: bool) -> None", "parameters": [ { "name": "enabled", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Enable or disable collection and lazily initialize Prometheus state.", "summary": "Enable or disable collection and lazily initialize Prometheus state.", "implementation": "Method `MetricsCollector.configure` updates `self._enabled`; calls `self._init_prometheus`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 98, "end_line": 105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L98-L105", "decorators": [], "calls": [ "self._init_prometheus" ], "state_reads": [ "self._lock", "self._prom", "self._init_prometheus" ], "state_writes": [ "self._enabled" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_init_prometheus", "qualname": "MetricsCollector._init_prometheus", "full_name": "vllm_mlx.metrics.MetricsCollector._init_prometheus", "kind": "method", "signature": "def _init_prometheus(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `MetricsCollector._init_prometheus` updates `self._prom`; calls `CollectorRegistry`, `Counter`, `Histogram`, `Gauge`.", "implementation": "Method `MetricsCollector._init_prometheus` updates `self._prom`; calls `CollectorRegistry`, `Counter`, `Histogram`, `Gauge`.", "documented": false, "public": false, "addressable": true, "line": 107, "end_line": 291, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L107-L291", "decorators": [], "calls": [ "CollectorRegistry", "Counter", "Histogram", "Gauge" ], "state_reads": [], "state_writes": [ "self._prom" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "track_inference", "qualname": "MetricsCollector.track_inference", "full_name": "vllm_mlx.metrics.MetricsCollector.track_inference", "kind": "method", "signature": "def track_inference(self, endpoint: str, *, stream: bool) -> InferenceTracker", "parameters": [ { "name": "endpoint", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "stream", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "InferenceTracker", "docstring": "Create request-scoped inference timing state for an endpoint.", "summary": "Create request-scoped inference timing state for an endpoint.", "implementation": "Method `MetricsCollector.track_inference` calls `InferenceTracker`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 293, "end_line": 298, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L293-L298", "decorators": [], "calls": [ "InferenceTracker" ], "state_reads": [ "self._enabled" ], "state_writes": [], "raises": [], "return_expressions": [ "InferenceTracker(None, endpoint, stream)", "InferenceTracker(self, endpoint, stream)" ], "awaits": false, "yields": false }, { "name": "observe_http_start", "qualname": "MetricsCollector.observe_http_start", "full_name": "vllm_mlx.metrics.MetricsCollector.observe_http_start", "kind": "method", "signature": "def observe_http_start(self, *, method: str, path: str) -> None", "parameters": [ { "name": "method", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "path", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Increment the in-flight request gauge for a normalized route.", "summary": "Increment the in-flight request gauge for a normalized route.", "implementation": "Method `MetricsCollector.observe_http_start` calls `self._prom['http_requests_in_flight'].labels(method=method, path=path).inc`, `self._prom['http_requests_in_flight'].labels`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 300, "end_line": 305, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L300-L305", "decorators": [], "calls": [ "self._prom['http_requests_in_flight'].labels(method=method, path=path).inc", "self._prom['http_requests_in_flight'].labels" ], "state_reads": [ "self._enabled", "self._prom" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "observe_http_finish", "qualname": "MetricsCollector.observe_http_finish", "full_name": "vllm_mlx.metrics.MetricsCollector.observe_http_finish", "kind": "method", "signature": "def observe_http_finish(self, *, method: str, path: str, status_code: int, duration: float) -> None", "parameters": [ { "name": "method", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "path", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "status_code", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "duration", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Record an HTTP result and decrement its in-flight gauge.", "summary": "Record an HTTP result and decrement its in-flight gauge.", "implementation": "Method `MetricsCollector.observe_http_finish` calls `self._prom['http_requests_in_flight'].labels(method=method, path=path).dec`, `self._prom['http_requests_in_flight'].labels`, `self._prom['http_requests_total'].labels(method=method, path=path, status_code=str(status_code)).inc`, `self._prom['http_requests_total'].labels`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 307, "end_line": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L307-L328", "decorators": [], "calls": [ "self._prom['http_requests_in_flight'].labels(method=method, path=path).dec", "self._prom['http_requests_in_flight'].labels", "self._prom['http_requests_total'].labels(method=method, path=path, status_code=str(status_code)).inc", "self._prom['http_requests_total'].labels", "str", "self._prom['http_request_duration_seconds'].labels(method=method, path=path).observe", "self._prom['http_request_duration_seconds'].labels" ], "state_reads": [ "self._enabled", "self._prom" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "observe_inference", "qualname": "MetricsCollector.observe_inference", "full_name": "vllm_mlx.metrics.MetricsCollector.observe_inference", "kind": "method", "signature": "def observe_inference(self, *, endpoint: str, stream: bool, result: str, duration: float, prompt_tokens: int, completion_tokens: int) -> None", "parameters": [ { "name": "endpoint", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "stream", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "result", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "duration", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "prompt_tokens", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "completion_tokens", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Record one terminal inference outcome, latency, and token totals.", "summary": "Record one terminal inference outcome, latency, and token totals.", "implementation": "Method `MetricsCollector.observe_inference` calls `_bool_str`, `self._prom['inference_requests_total'].labels(endpoint=endpoint, stream=stream_label, result=result).inc`, `self._prom['inference_requests_total'].labels`, `self._prom['inference_request_duration_seconds'].labels(endpoint=endpoint, stream=stream_label).observe`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 330, "end_line": 363, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L330-L363", "decorators": [], "calls": [ "_bool_str", "self._prom['inference_requests_total'].labels(endpoint=endpoint, stream=stream_label, result=result).inc", "self._prom['inference_requests_total'].labels", "self._prom['inference_request_duration_seconds'].labels(endpoint=endpoint, stream=stream_label).observe", "self._prom['inference_request_duration_seconds'].labels", "self._prom['prompt_tokens_total'].labels(endpoint=endpoint, stream=stream_label).inc", "self._prom['prompt_tokens_total'].labels", "self._prom['completion_tokens_total'].labels(endpoint=endpoint, stream=stream_label).inc", "self._prom['completion_tokens_total'].labels" ], "state_reads": [ "self._enabled", "self._prom" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "observe_ttft", "qualname": "MetricsCollector.observe_ttft", "full_name": "vllm_mlx.metrics.MetricsCollector.observe_ttft", "kind": "method", "signature": "def observe_ttft(self, *, endpoint: str, stream: bool, value: float) -> None", "parameters": [ { "name": "endpoint", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "stream", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "value", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Observe time to first token for a streaming or buffered request.", "summary": "Observe time to first token for a streaming or buffered request.", "implementation": "Method `MetricsCollector.observe_ttft` calls `self._prom['inference_ttft_seconds'].labels(endpoint=endpoint, stream=_bool_str(stream)).observe`, `self._prom['inference_ttft_seconds'].labels`, `_bool_str`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 365, "end_line": 373, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L365-L373", "decorators": [], "calls": [ "self._prom['inference_ttft_seconds'].labels(endpoint=endpoint, stream=_bool_str(stream)).observe", "self._prom['inference_ttft_seconds'].labels", "_bool_str" ], "state_reads": [ "self._enabled", "self._prom" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_update_engine_gauges", "qualname": "MetricsCollector._update_engine_gauges", "full_name": "vllm_mlx.metrics.MetricsCollector._update_engine_gauges", "kind": "method", "signature": "def _update_engine_gauges(self, *, engine: Any | None, mcp_manager: Any | None) -> None", "parameters": [ { "name": "engine", "kind": "keyword-only", "annotation": "Any | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "mcp_manager", "kind": "keyword-only", "annotation": "Any | None", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `MetricsCollector._update_engine_gauges` calls `engine.get_stats`, `self._prom['model_loaded'].set`, `stats.get`, `self._prom['engine_type'].labels(engine_type=engine_type).set`.", "implementation": "Method `MetricsCollector._update_engine_gauges` calls `engine.get_stats`, `self._prom['model_loaded'].set`, `stats.get`, `self._prom['engine_type'].labels(engine_type=engine_type).set`.", "documented": false, "public": false, "addressable": true, "line": 375, "end_line": 507, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L375-L507", "decorators": [], "calls": [ "engine.get_stats", "self._prom['model_loaded'].set", "stats.get", "self._prom['engine_type'].labels(engine_type=engine_type).set", "self._prom['engine_type'].labels", "self._prom['engine_is_mllm'].set", "self._prom['scheduler_waiting_requests'].set", "_coerce_int", "self._prom['scheduler_running_requests'].set", "self._prom['engine_steps_executed'].set", "self._prom['engine_uptime_seconds'].set", "_coerce_float", "self._prom['metal_memory_bytes'].labels(kind='active').set", "self._prom['metal_memory_bytes'].labels", "self._prom['metal_memory_bytes'].labels(kind='peak').set", "self._prom['metal_memory_bytes'].labels(kind='cache').set", "self._prom['cache_type'].labels(cache_type=candidate).set", "self._prom['cache_type'].labels", "isinstance", "self._prom['cache_entry_count'].set", "cache_stats.get", "self._prom['cache_hits'].set", "self._prom['cache_misses'].set", "self._prom['cache_evictions'].set", "self._prom['cache_hit_rate'].set", "self._prom['cache_utilization_ratio'].set", "self._prom['cache_tokens_saved'].set", "self._prom['cache_memory_bytes'].set", "self._prom['cache_memory_limit_bytes'].set", "get_registry().get_stats", "get_registry", "self._prom['model_registry_entries'].set", "registry_stats.get", "self._prom['model_registry_active_owners'].set", "list", "mcp_manager.get_server_status", "sum", "len", "mcp_manager.get_all_tools", "self._prom['mcp_connected_servers'].set", "self._prom['mcp_total_servers'].set", "self._prom['mcp_tools_available'].set" ], "state_reads": [ "self._prom" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "render_metrics", "qualname": "MetricsCollector.render_metrics", "full_name": "vllm_mlx.metrics.MetricsCollector.render_metrics", "kind": "method", "signature": "def render_metrics(self, *, engine: Any | None, mcp_manager: Any | None) -> tuple[bytes, str]", "parameters": [ { "name": "engine", "kind": "keyword-only", "annotation": "Any | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "mcp_manager", "kind": "keyword-only", "annotation": "Any | None", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "tuple[bytes, str]", "docstring": "Refresh runtime gauges and render Prometheus exposition bytes.\n\nRaises:\n RuntimeError: If metrics are disabled.", "summary": "Refresh runtime gauges and render Prometheus exposition bytes.", "implementation": "Method `MetricsCollector.render_metrics` calls `RuntimeError`, `self._init_prometheus`, `self._update_engine_gauges`, `self._prom['generate_latest']`; can raise `RuntimeError`; returns `(self._prom['generate_latest'](self._prom['registry']), self._prom['content_type'])`.", "documented": true, "public": true, "addressable": true, "line": 509, "end_line": 529, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/metrics.py#L509-L529", "decorators": [], "calls": [ "RuntimeError", "self._init_prometheus", "self._update_engine_gauges", "self._prom['generate_latest']" ], "state_reads": [ "self._enabled", "self._prom", "self._init_prometheus", "self._update_engine_gauges" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "(self._prom['generate_latest'](self._prom['registry']), self._prom['content_type'])" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.mllm_batch_generator", "path": "vllm_mlx/mllm_batch_generator.py", "page_path": "reference/api/vllm_mlx/mllm_batch_generator.md", "docstring": "MLLM Batch Generator for multimodal continuous batching.\n\nThis module implements continuous batching for Multimodal Language Models (MLLMs)\nlike Qwen3-VL, following the same architecture as LLM continuous batching but\nadapted for vision models.\n\nKey insight: VLM models have a `model.language_model` which is a standard LLM.\nAfter the initial forward pass with vision encoding, text generation uses only\nthe language model - which CAN be batched using the same BatchKVCache pattern.\n\nArchitecture:\n1. Vision inputs are processed per-request (not batched)\n2. Initial VLM forward pass extracts cross-attention states / encoder outputs\n3. Language model generation is batched using BatchKVCache (like LLM batching)", "summary": "MLLM Batch Generator for multimodal continuous batching.", "line_count": 3073, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1-L3073", "members": [ "logger", "_processors_can_retire", "_mark_mtp_attempts_on_primary_responses", "_drop_retired_processors", "_request_uses_stochastic_sampling", "_sampling_logprobs", "_residual_logprobs", "_accept_sampled_draft", "PrefillAbortedError", "_cache_eval_tensors", "_eval_prompt_cache", "MLLMBatchRequest", "MLLMBatchResponse", "MLLMBatch", "MLLMBatchStats", "_left_pad_prompts", "MLLMBatchGenerator", "install_mtp_mllm", "install_chunked_prefill_mllm" ], "symbols": [ { "name": "_processors_can_retire", "qualname": "_processors_can_retire", "full_name": "vllm_mlx.mllm_batch_generator._processors_can_retire", "kind": "function", "signature": "def _processors_can_retire(processors: Optional[List[Callable]]) -> bool", "parameters": [ { "name": "processors", "kind": "positional or keyword", "annotation": "Optional[List[Callable]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "True when any processor advertises a retire-to-content transition.", "summary": "True when any processor advertises a retire-to-content transition.", "implementation": "Function `_processors_can_retire` calls `os.getenv`, `bool`, `any`, `isinstance`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 37, "end_line": 43, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L37-L43", "decorators": [], "calls": [ "os.getenv", "bool", "any", "isinstance", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "bool(processors) and any((isinstance(getattr(p, 'is_retired', None), bool) for p in processors))" ], "awaits": false, "yields": false }, { "name": "_mark_mtp_attempts_on_primary_responses", "qualname": "_mark_mtp_attempts_on_primary_responses", "full_name": "vllm_mlx.mllm_batch_generator._mark_mtp_attempts_on_primary_responses", "kind": "function", "signature": "def _mark_mtp_attempts_on_primary_responses(responses: List['MLLMBatchResponse'], attempted_drafts_by_uid: Dict[int, int]) -> None", "parameters": [ { "name": "responses", "kind": "positional or keyword", "annotation": "List['MLLMBatchResponse']", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attempted_drafts_by_uid", "kind": "positional or keyword", "annotation": "Dict[int, int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Mark only responses from steps that actually attempted MTP drafts.", "summary": "Mark only responses from steps that actually attempted MTP drafts.", "implementation": "Function `_mark_mtp_attempts_on_primary_responses` calls `attempted_drafts_by_uid.pop`, `attempted_drafts_by_uid.clear`.", "documented": true, "public": false, "addressable": true, "line": 46, "end_line": 57, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L46-L57", "decorators": [], "calls": [ "attempted_drafts_by_uid.pop", "attempted_drafts_by_uid.clear" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_drop_retired_processors", "qualname": "_drop_retired_processors", "full_name": "vllm_mlx.mllm_batch_generator._drop_retired_processors", "kind": "function", "signature": "def _drop_retired_processors(processors: Optional[List[Callable]]) -> tuple[Optional[List[Callable]], int]", "parameters": [ { "name": "processors", "kind": "positional or keyword", "annotation": "Optional[List[Callable]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[Optional[List[Callable]], int]", "docstring": "Drop retire-capable processors that have completed their work.", "summary": "Drop retire-capable processors that have completed their work.", "implementation": "Function `_drop_retired_processors` calls `getattr`, `remaining.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 60, "end_line": 74, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L60-L74", "decorators": [], "calls": [ "getattr", "remaining.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(processors, 0)", "(remaining or None, retired_count)" ], "awaits": false, "yields": false }, { "name": "_request_uses_stochastic_sampling", "qualname": "_request_uses_stochastic_sampling", "full_name": "vllm_mlx.mllm_batch_generator._request_uses_stochastic_sampling", "kind": "function", "signature": "def _request_uses_stochastic_sampling(request: Any) -> bool", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return whether a request needs sampler-aware speculative verification.\n\nGreedy (temperature 0) requests are excluded regardless of top_p/top_k/\nmin_p: _sampling_logprobs() collapses to an argmax delta distribution for\ntemperature 0 and never applies those filters, so a greedy request left at\na non-default top_p/top_k/min_p is not actually stochastic.", "summary": "Return whether a request needs sampler-aware speculative verification.", "implementation": "Function `_request_uses_stochastic_sampling` calls `getattr`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 77, "end_line": 92, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L77-L92", "decorators": [], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "getattr(request, 'top_p', 1.0) < 1.0 or getattr(request, 'top_k', 0) != 0 or getattr(request, 'min_p', 0.0) != 0.0" ], "awaits": false, "yields": false }, { "name": "_sampling_logprobs", "qualname": "_sampling_logprobs", "full_name": "vllm_mlx.mllm_batch_generator._sampling_logprobs", "kind": "function", "signature": "def _sampling_logprobs(logits: mx.array, request: Any) -> mx.array", "parameters": [ { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Match mlx-lm's request sampler in log-probability space.\n\nSpeculative decoding compares the post-filter distributions, not the raw\ntarget and draft logits. Keep this transformation here rather than reusing\na greedy verifier for sampled requests.", "summary": "Match mlx-lm's request sampler in log-probability space.", "implementation": "Function `_sampling_logprobs` calls `getattr`, `mx.logsumexp`, `mx.argmax`, `mx.full`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 95, "end_line": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L95-L123", "decorators": [], "calls": [ "getattr", "mx.logsumexp", "mx.argmax", "mx.full", "float", "mx.put_along_axis", "apply_top_p", "apply_min_p", "apply_top_k" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.put_along_axis(result, token[:, None], 0.0, axis=-1)", "logprobs - mx.logsumexp(logprobs, axis=-1, keepdims=True)" ], "awaits": false, "yields": false }, { "name": "_residual_logprobs", "qualname": "_residual_logprobs", "full_name": "vllm_mlx.mllm_batch_generator._residual_logprobs", "kind": "function", "signature": "def _residual_logprobs(target_logprobs: mx.array, draft_logprobs: mx.array) -> mx.array", "parameters": [ { "name": "target_logprobs", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "draft_logprobs", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Return the normalized residual max(target - draft, 0) distribution.", "summary": "Return the normalized residual max(target - draft, 0) distribution.", "implementation": "Function `_residual_logprobs` calls `mx.maximum`, `mx.exp`, `mx.sum`, `mx.where`; returns `mx.where(mass > 1e-12, normalized, fallback)`.", "documented": true, "public": false, "addressable": true, "line": 126, "end_line": 139, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L126-L139", "decorators": [], "calls": [ "mx.maximum", "mx.exp", "mx.sum", "mx.where", "mx.log", "float" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.where(mass > 1e-12, normalized, fallback)" ], "awaits": false, "yields": false }, { "name": "_accept_sampled_draft", "qualname": "_accept_sampled_draft", "full_name": "vllm_mlx.mllm_batch_generator._accept_sampled_draft", "kind": "function", "signature": "def _accept_sampled_draft(target_logprob: float, draft_logprob: float, uniform_draw: float) -> bool", "parameters": [ { "name": "target_logprob", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "draft_logprob", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "uniform_draw", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Apply the exact min(1, p/q) stochastic speculative acceptance rule.", "summary": "Apply the exact min(1, p/q) stochastic speculative acceptance rule.", "implementation": "Function `_accept_sampled_draft` calls `math.log`, `max`; returns `log_acceptance >= 0.0 or math.log(max(uniform_draw, 1e-35)) < log_acceptance`.", "documented": true, "public": false, "addressable": true, "line": 142, "end_line": 149, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L142-L149", "decorators": [], "calls": [ "math.log", "max" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "log_acceptance >= 0.0 or math.log(max(uniform_draw, 1e-35)) < log_acceptance" ], "awaits": false, "yields": false }, { "name": "PrefillAbortedError", "qualname": "PrefillAbortedError", "full_name": "vllm_mlx.mllm_batch_generator.PrefillAbortedError", "kind": "class", "signature": "class PrefillAbortedError(Exception)", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "PrefillAbortedError", "docstring": "Raised when a prefill is aborted due to client disconnect.", "summary": "Raised when a prefill is aborted due to client disconnect.", "implementation": "Class `PrefillAbortedError` derives from `Exception` and declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 152, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L152-L157", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "PrefillAbortedError.__init__", "full_name": "vllm_mlx.mllm_batch_generator.PrefillAbortedError.__init__", "kind": "method", "signature": "def __init__(self, request_id: str)", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `PrefillAbortedError.__init__` updates `self.request_id`; calls `super().__init__`, `super`.", "implementation": "Method `PrefillAbortedError.__init__` updates `self.request_id`; calls `super().__init__`, `super`.", "documented": false, "public": false, "addressable": true, "line": 155, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L155-L157", "decorators": [], "calls": [ "super().__init__", "super" ], "state_reads": [], "state_writes": [ "self.request_id" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_cache_eval_tensors", "qualname": "_cache_eval_tensors", "full_name": "vllm_mlx.mllm_batch_generator._cache_eval_tensors", "kind": "function", "signature": "def _cache_eval_tensors(cache: List[Any]) -> List[Any]", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "List[Any]", "docstring": "Return realized tensors that break lazy cache graphs between chunks.", "summary": "Return realized tensors that break lazy cache graphs between chunks.", "implementation": "Function `_cache_eval_tensors` calls `getattr`, `tensors.append`, `isinstance`, `tensors.extend`; returns `tensors`.", "documented": true, "public": false, "addressable": true, "line": 160, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L160-L183", "decorators": [], "calls": [ "getattr", "tensors.append", "isinstance", "tensors.extend" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tensors" ], "awaits": false, "yields": false }, { "name": "_eval_prompt_cache", "qualname": "_eval_prompt_cache", "full_name": "vllm_mlx.mllm_batch_generator._eval_prompt_cache", "kind": "function", "signature": "def _eval_prompt_cache(cache: List[Any]) -> None", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Evaluate all cache tensors used by hybrid chunked prefill.", "summary": "Evaluate all cache tensors used by hybrid chunked prefill.", "implementation": "Function `_eval_prompt_cache` calls `_cache_eval_tensors`, `mx.eval`.", "documented": true, "public": false, "addressable": true, "line": 186, "end_line": 190, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L186-L190", "decorators": [], "calls": [ "_cache_eval_tensors", "mx.eval" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLLMBatchRequest", "qualname": "MLLMBatchRequest", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchRequest", "kind": "class", "signature": "class MLLMBatchRequest", "parameters": [ { "name": "uid", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "request_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "images", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "videos", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "audio", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_tokens", "kind": "field", "annotation": "int", "default": "256", "required": false, "description": "Optional constructor field; defaults to `256`." }, { "name": "temperature", "kind": "field", "annotation": "float", "default": "0.7", "required": false, "description": "Optional constructor field; defaults to `0.7`." }, { "name": "top_p", "kind": "field", "annotation": "float", "default": "0.9", "required": false, "description": "Optional constructor field; defaults to `0.9`." }, { "name": "top_k", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "min_p", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "presence_penalty", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "repetition_penalty", "kind": "field", "annotation": "float", "default": "1.0", "required": false, "description": "Optional constructor field; defaults to `1.0`." }, { "name": "logits_processors", "kind": "field", "annotation": "Optional[List[Callable]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "input_ids", "kind": "field", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "pixel_values", "kind": "field", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "attention_mask", "kind": "field", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "image_grid_thw", "kind": "field", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "extra_kwargs", "kind": "field", "annotation": "Dict[str, Any]", "default": "field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=dict)`." }, { "name": "is_text_only", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "num_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "output_tokens", "kind": "field", "annotation": "List[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "vision_encoded", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "cross_attention_states", "kind": "field", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "encoder_outputs", "kind": "field", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MLLMBatchRequest", "docstring": "Request data for MLLM batch processing.\n\nContains all information needed to process a multimodal request\nwithin the batch generator.", "summary": "Request data for MLLM batch processing.", "implementation": "Class `MLLMBatchRequest` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 194, "end_line": 237, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L194-L237", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLLMBatchResponse", "qualname": "MLLMBatchResponse", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchResponse", "kind": "class", "signature": "class MLLMBatchResponse", "parameters": [ { "name": "uid", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "request_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "token", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "logprobs", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "finish_reason", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prompt_cache", "kind": "field", "annotation": "Optional[Callable[[], List[Any]]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "from_draft", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "mtp_attempted", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "mtp_attempted_count", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "MLLMBatchResponse", "docstring": "Response from a batch generation step.\n\nContains the generated token and metadata for a single request.", "summary": "Response from a batch generation step.", "implementation": "Class `MLLMBatchResponse` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 241, "end_line": 256, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L241-L256", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLLMBatch", "qualname": "MLLMBatch", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatch", "kind": "class", "signature": "class MLLMBatch", "parameters": [ { "name": "uids", "kind": "field", "annotation": "List[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "request_ids", "kind": "field", "annotation": "List[str]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "y", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "logprobs", "kind": "field", "annotation": "List[mx.array]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "max_tokens", "kind": "field", "annotation": "List[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "num_tokens", "kind": "field", "annotation": "List[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "cache", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "requests", "kind": "field", "annotation": "List[MLLMBatchRequest]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "logits_processors", "kind": "field", "annotation": "Optional[List[Optional[List[Callable]]]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "samplers", "kind": "field", "annotation": "Optional[List[Optional[Callable]]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MLLMBatch", "docstring": "Represents an active batch of MLLM requests.\n\nManages the batch state including tokens, caches, and metadata\nfor all requests being processed together.", "summary": "Represents an active batch of MLLM requests.", "implementation": "Class `MLLMBatch` declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 260, "end_line": 392, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L260-L392", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "MLLMBatch.__len__", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatch.__len__", "kind": "method", "signature": "def __len__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "", "summary": "Method `MLLMBatch.__len__` calls `len`; returns `len(self.uids)`.", "implementation": "Method `MLLMBatch.__len__` calls `len`; returns `len(self.uids)`.", "documented": false, "public": false, "addressable": true, "line": 279, "end_line": 280, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L279-L280", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.uids" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.uids)" ], "awaits": false, "yields": false }, { "name": "filter", "qualname": "MLLMBatch.filter", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatch.filter", "kind": "method", "signature": "def filter(self, keep_idx: List[int]) -> None", "parameters": [ { "name": "keep_idx", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Indices of requests to keep" } ], "return_annotation": "None", "docstring": "Filter batch to keep only requests at specified indices.\n\nArgs:\n keep_idx: Indices of requests to keep", "summary": "Filter batch to keep only requests at specified indices.", "implementation": "Method `MLLMBatch.filter` updates `self.uids`, `self.request_ids`, `self.logprobs`, `self.max_tokens`; calls `mx.array`, `hasattr`, `c.filter`.", "documented": true, "public": true, "addressable": true, "line": 282, "end_line": 306, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L282-L306", "decorators": [], "calls": [ "mx.array", "hasattr", "c.filter" ], "state_reads": [ "self.uids", "self.request_ids", "self.logprobs", "self.max_tokens", "self.num_tokens", "self.requests", "self.logits_processors", "self.samplers", "self.y", "self.cache" ], "state_writes": [ "self.uids", "self.request_ids", "self.logprobs", "self.max_tokens", "self.num_tokens", "self.requests", "self.logits_processors", "self.samplers", "self.y" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extend", "qualname": "MLLMBatch.extend", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatch.extend", "kind": "method", "signature": "def extend(self, other: 'MLLMBatch') -> None", "parameters": [ { "name": "other", "kind": "positional or keyword", "annotation": "'MLLMBatch'", "default": "", "required": true, "description": "Batch to merge into this one" } ], "return_annotation": "None", "docstring": "Extend this batch with another batch.\n\nArgs:\n other: Batch to merge into this one", "summary": "Extend this batch with another batch.", "implementation": "Method `MLLMBatch.extend` updates `self.y`, `self.logits_processors`, `self.samplers`; calls `self.uids.extend`, `self.request_ids.extend`, `mx.concatenate`, `self.logprobs.extend`.", "documented": true, "public": true, "addressable": true, "line": 308, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L308-L351", "decorators": [], "calls": [ "self.uids.extend", "self.request_ids.extend", "mx.concatenate", "self.logprobs.extend", "self.num_tokens.extend", "self.max_tokens.extend", "self.requests.extend", "len", "list", "zip", "hasattr", "c.empty", "c.extend", "logger.warning" ], "state_reads": [ "self.uids.extend", "self.uids", "self.request_ids.extend", "self.request_ids", "self.y", "self.logprobs.extend", "self.logprobs", "self.num_tokens.extend", "self.num_tokens", "self.max_tokens.extend", "self.max_tokens", "self.requests.extend", "self.requests", "self.logits_processors", "self.samplers", "self.cache" ], "state_writes": [ "self.y", "self.logits_processors", "self.samplers" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_cache", "qualname": "MLLMBatch.extract_cache", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatch.extract_cache", "kind": "method", "signature": "def extract_cache(self, idx: int) -> List[Any]", "parameters": [ { "name": "idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "List[Any]", "docstring": "Extract cache for a single request (for prefix caching).\n\nHandles BatchRotatingKVCache negative left_padding bug:\nduring generation with rotation, left_padding becomes negative,\ncausing extract() to use Python negative indexing and truncate\nthe buffer to only generation tokens instead of the full window.", "summary": "Extract cache for a single request (for prefix caching).", "implementation": "Method `MLLMBatch.extract_cache` calls `hasattr`, `result.append`, `isinstance`, `RotatingKVCache`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 353, "end_line": 392, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L353-L392", "decorators": [], "calls": [ "hasattr", "result.append", "isinstance", "RotatingKVCache", "max", "c.left_padding[idx].item", "c.offset[idx].item", "mx.roll", "mx.contiguous", "getattr", "c.extract" ], "state_reads": [ "self.cache" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "MLLMBatchStats", "qualname": "MLLMBatchStats", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchStats", "kind": "class", "signature": "class MLLMBatchStats", "parameters": [], "return_annotation": "MLLMBatchStats", "docstring": "Statistics for MLLM batch generation.", "summary": "Statistics for MLLM batch generation.", "implementation": "Class `MLLMBatchStats` declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 395, "end_line": 436, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L395-L436", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLLMBatchStats.__init__", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchStats.__init__", "kind": "method", "signature": "def __init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `MLLMBatchStats.__init__` updates `self.prompt_tokens`, `self.prompt_time`, `self.generation_tokens`, `self.generation_time`.", "implementation": "Method `MLLMBatchStats.__init__` updates `self.prompt_tokens`, `self.prompt_time`, `self.generation_tokens`, `self.generation_time`.", "documented": false, "public": false, "addressable": true, "line": 398, "end_line": 405, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L398-L405", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.prompt_tokens", "self.prompt_time", "self.generation_tokens", "self.generation_time", "self.vision_encoding_time", "self.num_images_processed", "self.peak_memory" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "prompt_tps", "qualname": "MLLMBatchStats.prompt_tps", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchStats.prompt_tps", "kind": "method", "signature": "def prompt_tps(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Return measured multimodal prompt throughput in tokens per second.", "summary": "Return measured multimodal prompt throughput in tokens per second.", "implementation": "Method `MLLMBatchStats.prompt_tps` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 408, "end_line": 413, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L408-L413", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.prompt_time", "self.prompt_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "0", "self.prompt_tokens / self.prompt_time" ], "awaits": false, "yields": false }, { "name": "generation_tps", "qualname": "MLLMBatchStats.generation_tps", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchStats.generation_tps", "kind": "method", "signature": "def generation_tps(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Return measured decode throughput in tokens per second.", "summary": "Return measured decode throughput in tokens per second.", "implementation": "Method `MLLMBatchStats.generation_tps` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 416, "end_line": 421, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L416-L421", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.generation_time", "self.generation_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "0", "self.generation_tokens / self.generation_time" ], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "MLLMBatchStats.to_dict", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchStats.to_dict", "kind": "method", "signature": "def to_dict(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Return token, timing, vision, and peak-memory statistics.", "summary": "Return token, timing, vision, and peak-memory statistics.", "implementation": "Method `MLLMBatchStats.to_dict` returns `{'prompt_tokens': self.prompt_tokens, 'prompt_time': self.prompt_time, 'prompt_tps': self.prompt_tps, 'generation_token…`.", "documented": true, "public": true, "addressable": true, "line": 423, "end_line": 436, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L423-L436", "decorators": [], "calls": [], "state_reads": [ "self.prompt_tokens", "self.prompt_time", "self.prompt_tps", "self.generation_tokens", "self.generation_time", "self.generation_tps", "self.vision_encoding_time", "self.num_images_processed", "self.peak_memory" ], "state_writes": [], "raises": [], "return_expressions": [ "{'prompt_tokens': self.prompt_tokens, 'prompt_time': self.prompt_time, 'prompt_tps': self.prompt_tps, 'generation_token…" ], "awaits": false, "yields": false }, { "name": "_left_pad_prompts", "qualname": "_left_pad_prompts", "full_name": "vllm_mlx.mllm_batch_generator._left_pad_prompts", "kind": "function", "signature": "def _left_pad_prompts(prompts: List[List[int]], max_length: Optional[int]=None) -> mx.array", "parameters": [ { "name": "prompts", "kind": "positional or keyword", "annotation": "List[List[int]]", "default": "", "required": true, "description": "List of token lists" }, { "name": "max_length", "kind": "positional or keyword", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Target length (computed if not provided)" } ], "return_annotation": "mx.array", "docstring": "Left-pad prompts to uniform length.\n\nArgs:\n prompts: List of token lists\n max_length: Target length (computed if not provided)\n\nReturns:\n Padded prompts as mx.array [batch_size, seq_len]", "summary": "Left-pad prompts to uniform length.", "implementation": "Function `_left_pad_prompts` calls `max`, `len`, `mx.array`, `list`; returns `mx.array([[0] * (max_length - len(p)) + list(p) for p in prompts])`.", "documented": true, "public": false, "addressable": true, "line": 439, "end_line": 454, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L439-L454", "decorators": [], "calls": [ "max", "len", "mx.array", "list" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.array([[0] * (max_length - len(p)) + list(p) for p in prompts])" ], "awaits": false, "yields": false }, { "name": "MLLMBatchGenerator", "qualname": "MLLMBatchGenerator", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator", "kind": "class", "signature": "class MLLMBatchGenerator", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "nn.Module", "default": "", "required": true, "description": "The VLM model (must have model.language_model)" }, { "name": "processor", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM processor for tokenization and image processing" }, { "name": "mm_processor", "kind": "positional or keyword", "annotation": "Optional[MultimodalProcessor]", "default": "None", "required": false, "description": "Optional MultimodalProcessor for input preparation" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Default max tokens per request" }, { "name": "stop_tokens", "kind": "positional or keyword", "annotation": "Optional[set]", "default": "None", "required": false, "description": "Set of stop token IDs" }, { "name": "sampler", "kind": "positional or keyword", "annotation": "Optional[Callable[[mx.array], mx.array]]", "default": "None", "required": false, "description": "Sampling function (default: argmax)" }, { "name": "prefill_batch_size", "kind": "positional or keyword", "annotation": "int", "default": "4", "required": false, "description": "Max requests to prefill together" }, { "name": "completion_batch_size", "kind": "positional or keyword", "annotation": "int", "default": "16", "required": false, "description": "Max requests for completion batching" }, { "name": "prefill_step_size", "kind": "positional or keyword", "annotation": "int", "default": "1024", "required": false, "description": "Tokens to process per prefill step" }, { "name": "enable_vision_cache", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Enable vision embedding caching" }, { "name": "vision_cache_size", "kind": "positional or keyword", "annotation": "int", "default": "100", "required": false, "description": "Max entries in vision cache" }, { "name": "prefix_cache_config", "kind": "positional or keyword", "annotation": "Optional[MemoryCacheConfig]", "default": "None", "required": false, "description": "Config for KV prefix cache (text-only requests)" }, { "name": "max_kv_size", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Maximum KV cache size per sequence (0 = unbounded)" } ], "return_annotation": "MLLMBatchGenerator", "docstring": "Batch generator for Vision Language Models.\n\nThis class manages continuous batching for MLLM requests:\n\n1. Vision Encoding Phase:\n - Process images/videos through vision encoder (per-request)\n - Extract vision features and merge with text embeddings\n - Store cross-attention states for language model\n\n2. Language Generation Phase:\n - Use language model with BatchKVCache for batched generation\n - Generate tokens for all requests simultaneously\n - Same pattern as LLM BatchGenerator\n\nExample:\n >>> generator = MLLMBatchGenerator(model, processor)\n >>> uids = generator.insert([request1, request2])\n >>> while responses := generator.next():\n ... for resp in responses:\n ... print(f\"Request {resp.request_id}: token={resp.token}\")", "summary": "Batch generator for Vision Language Models.", "implementation": "Class `MLLMBatchGenerator` declares 26 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 457, "end_line": 2042, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L457-L2042", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLLMBatchGenerator.__init__", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.__init__", "kind": "method", "signature": "def __init__(self, model: nn.Module, processor: Any, mm_processor: Optional[MultimodalProcessor]=None, max_tokens: int=256, stop_tokens: Optional[set]=None, sampler: Optional[Callable[[mx.array], mx.array]]=None, prefill_batch_size: int=4, completion_batch_size: int=16, prefill_step_size: int=1024, enable_vision_cache: bool=True, vision_cache_size: int=100, prefix_cache_config: Optional[MemoryCacheConfig]=None, max_kv_size: int=0)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "nn.Module", "default": "", "required": true, "description": "The VLM model (must have model.language_model)" }, { "name": "processor", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM processor for tokenization and image processing" }, { "name": "mm_processor", "kind": "positional or keyword", "annotation": "Optional[MultimodalProcessor]", "default": "None", "required": false, "description": "Optional MultimodalProcessor for input preparation" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Default max tokens per request" }, { "name": "stop_tokens", "kind": "positional or keyword", "annotation": "Optional[set]", "default": "None", "required": false, "description": "Set of stop token IDs" }, { "name": "sampler", "kind": "positional or keyword", "annotation": "Optional[Callable[[mx.array], mx.array]]", "default": "None", "required": false, "description": "Sampling function (default: argmax)" }, { "name": "prefill_batch_size", "kind": "positional or keyword", "annotation": "int", "default": "4", "required": false, "description": "Max requests to prefill together" }, { "name": "completion_batch_size", "kind": "positional or keyword", "annotation": "int", "default": "16", "required": false, "description": "Max requests for completion batching" }, { "name": "prefill_step_size", "kind": "positional or keyword", "annotation": "int", "default": "1024", "required": false, "description": "Tokens to process per prefill step" }, { "name": "enable_vision_cache", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Enable vision embedding caching" }, { "name": "vision_cache_size", "kind": "positional or keyword", "annotation": "int", "default": "100", "required": false, "description": "Max entries in vision cache" }, { "name": "prefix_cache_config", "kind": "positional or keyword", "annotation": "Optional[MemoryCacheConfig]", "default": "None", "required": false, "description": "Config for KV prefix cache (text-only requests)" }, { "name": "max_kv_size", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Maximum KV cache size per sequence (0 = unbounded)" } ], "return_annotation": "not annotated", "docstring": "Initialize MLLM batch generator.\n\nArgs:\n model: The VLM model (must have model.language_model)\n processor: The VLM processor for tokenization and image processing\n mm_processor: Optional MultimodalProcessor for input preparation\n max_tokens: Default max tokens per request\n stop_tokens: Set of stop token IDs\n sampler: Sampling function (default: argmax)\n prefill_batch_size: Max requests to prefill together\n completion_batch_size: Max requests for completion batching\n prefill_step_size: Tokens to process per prefill step\n enable_vision_cache: Enable vision embedding caching\n vision_cache_size: Max entries in vision cache\n prefix_cache_config: Config for KV prefix cache (text-only requests)\n max_kv_size: Maximum KV cache size per sequence (0 = unbounded)", "summary": "Initialize MLLM batch generator.", "implementation": "Method `MLLMBatchGenerator.__init__` updates `self.model`, `self.processor`, `self.mm_processor`, `self.max_kv_size`; calls `getattr`, `hasattr`, `logger.info`, `logger.warning`.", "documented": true, "public": false, "addressable": true, "line": 484, "end_line": 632, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L484-L632", "decorators": [], "calls": [ "getattr", "hasattr", "logger.info", "logger.warning", "patch_qwen35_attention_for_batching", "patch_gemma4_attention_for_batching", "patch_glm4v_moe_for_batching", "set", "max", "MLLMBatchStats", "threading.Lock", "VisionEmbeddingCache", "MemoryAwarePrefixCache", "self._normalize_chat_template_for_prefix_cache", "self._compute_think_suffix_len", "mx.new_stream", "mx.default_device", "mx.metal.is_available", "mx.set_wired_limit", "mx.device_info" ], "state_reads": [ "self.is_vlm", "self.language_model", "self._normalize_chat_template_for_prefix_cache", "self._compute_think_suffix_len" ], "state_writes": [ "self.model", "self.processor", "self.mm_processor", "self.max_kv_size", "self.language_model", "self.is_vlm", "self.max_tokens", "self.stop_tokens", "self.sampler", "self.prefill_batch_size", "self.completion_batch_size", "self.prefill_step_size", "self.unprocessed_requests", "self.active_batch", "self.uid_counter", "self._stats", "self._pending_error_responses", "self._prefill_progress", "self._aborted_request_ids", "self._pending_removal_uids", "self._pending_removal_lock", "self.vision_cache", "self.prefix_cache", "self._think_suffix_len", "self._old_wired_limit" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_normalize_chat_template_for_prefix_cache", "qualname": "MLLMBatchGenerator._normalize_chat_template_for_prefix_cache", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._normalize_chat_template_for_prefix_cache", "kind": "method", "signature": "def _normalize_chat_template_for_prefix_cache(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Patch chat template so historical assistant turns are prefix-stable.\n\nQwen3.5's chat template computes ``last_query_index`` — the position\nof the last non-tool-response user message — and conditionally wraps\nassistant turns after that index in ``...\\n\\n\\n``.\nWhen a new user text message is appended, ``last_query_index`` jumps\nforward, retroactively removing these ```` wrappers from\nearlier assistant turns. This shifts tokens mid-sequence and breaks\nprefix cache.\n\nFix: replace the conditional with the plain (ELSE) branch so ALL\nhistorical assistant messages use ``<|im_start|>assistant\\ncontent``\nwithout any injected ```` block. The generation prompt still\nadds ``\\n`` at the very end, so the model generates thinking.", "summary": "Patch chat template so historical assistant turns are prefix-stable.", "implementation": "Method `MLLMBatchGenerator._normalize_chat_template_for_prefix_cache` updates `self.processor.chat_template`; calls `getattr`, `re.sub`, `hasattr`, `logger.info`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 634, "end_line": 697, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L634-L697", "decorators": [], "calls": [ "getattr", "re.sub", "hasattr", "logger.info", "logger.debug" ], "state_reads": [ "self.prefix_cache", "self.processor" ], "state_writes": [ "self.processor.chat_template" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_compute_think_suffix_len", "qualname": "MLLMBatchGenerator._compute_think_suffix_len", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._compute_think_suffix_len", "kind": "method", "signature": "def _compute_think_suffix_len(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Compute how many extra tokens enable_thinking=True adds at the END.\n\nCompares the generation prompt suffix with and without\n``enable_thinking`` to find the think-tag suffix length\n(typically ``\\n`` = 2 tokens for Qwen3/Qwen3.5).\n\nReturns 0 if the template doesn't support ``enable_thinking``.", "summary": "Compute how many extra tokens enable_thinking=True adds at the END.", "implementation": "Method `MLLMBatchGenerator._compute_think_suffix_len` calls `getattr`, `hasattr`, `applicator.apply_chat_template`, `text_with.endswith`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 699, "end_line": 758, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L699-L758", "decorators": [], "calls": [ "getattr", "hasattr", "applicator.apply_chat_template", "text_with.endswith", "text_without.endswith", "tokenizer.encode", "len", "logger.info", "tag.strip", "max" ], "state_reads": [ "self.processor" ], "state_writes": [], "raises": [], "return_expressions": [ "0", "max(0, suffix_len)" ], "awaits": false, "yields": false }, { "name": "close", "qualname": "MLLMBatchGenerator.close", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.close", "kind": "method", "signature": "def close(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Release resources and reset wired limit.", "summary": "Release resources and reset wired limit.", "implementation": "Method `MLLMBatchGenerator.close` updates `self._old_wired_limit`; calls `mx.synchronize`, `mx.set_wired_limit`.", "documented": true, "public": true, "addressable": true, "line": 760, "end_line": 765, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L760-L765", "decorators": [], "calls": [ "mx.synchronize", "mx.set_wired_limit" ], "state_reads": [ "self._old_wired_limit" ], "state_writes": [ "self._old_wired_limit" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "abort_prefill", "qualname": "MLLMBatchGenerator.abort_prefill", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.abort_prefill", "kind": "method", "signature": "def abort_prefill(self, request_id: str) -> None", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Signal that a request's prefill should be aborted.\n\nCalled from the event loop thread when a client disconnects.\nThe prefill loop checks this set between chunks and raises\nPrefillAbortedError to exit early.", "summary": "Signal that a request's prefill should be aborted.", "implementation": "Method `MLLMBatchGenerator.abort_prefill` calls `self._aborted_request_ids.add`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 767, "end_line": 775, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L767-L775", "decorators": [], "calls": [ "self._aborted_request_ids.add", "logger.info" ], "state_reads": [ "self._aborted_request_ids.add", "self._aborted_request_ids" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "schedule_removal", "qualname": "MLLMBatchGenerator.schedule_removal", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.schedule_removal", "kind": "method", "signature": "def schedule_removal(self, uids: List[int]) -> None", "parameters": [ { "name": "uids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Thread-safe deferred removal of UIDs from the batch.\n\nSafe to call from any thread (typically the event loop during\nclient-disconnect cleanup). The actual `remove()`, which creates\n``mx.array`` instances and filters the KV cache, runs on the\nscheduler thread via :meth:`process_pending_removals` at the next\nbatch boundary. This avoids the Metal ``encodeSignalEvent:\nuncommitted encoder`` crash that occurs when two threads submit\nGPU work on the same stream concurrently.", "summary": "Thread-safe deferred removal of UIDs from the batch.", "implementation": "Method `MLLMBatchGenerator.schedule_removal` calls `self._pending_removal_uids.update`.", "documented": true, "public": true, "addressable": true, "line": 777, "end_line": 789, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L777-L789", "decorators": [], "calls": [ "self._pending_removal_uids.update" ], "state_reads": [ "self._pending_removal_lock", "self._pending_removal_uids.update", "self._pending_removal_uids" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "process_pending_removals", "qualname": "MLLMBatchGenerator.process_pending_removals", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.process_pending_removals", "kind": "method", "signature": "def process_pending_removals(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Remove any UIDs enqueued via :meth:`schedule_removal`.\n\nMUST be called from the scheduler thread only, at a safe point\n(e.g. the start of :meth:`MLLMScheduler.step` before any forward\npass has been issued). Safe to call even when the queue is\nempty (no-op).", "summary": "Remove any UIDs enqueued via :meth:`schedule_removal`.", "implementation": "Method `MLLMBatchGenerator.process_pending_removals` updates `self._pending_removal_uids`; calls `set`, `list`, `self.remove`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 791, "end_line": 808, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L791-L808", "decorators": [], "calls": [ "set", "list", "self.remove" ], "state_reads": [ "self._pending_removal_lock", "self._pending_removal_uids", "self.remove" ], "state_writes": [ "self._pending_removal_uids" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "__del__", "qualname": "MLLMBatchGenerator.__del__", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.__del__", "kind": "method", "signature": "def __del__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `MLLMBatchGenerator.__del__` calls `self.close`.", "implementation": "Method `MLLMBatchGenerator.__del__` calls `self.close`.", "documented": false, "public": false, "addressable": true, "line": 810, "end_line": 814, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L810-L814", "decorators": [], "calls": [ "self.close" ], "state_reads": [ "self.close" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "insert", "qualname": "MLLMBatchGenerator.insert", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.insert", "kind": "method", "signature": "def insert(self, requests: List[MLLMBatchRequest]) -> List[int]", "parameters": [ { "name": "requests", "kind": "positional or keyword", "annotation": "List[MLLMBatchRequest]", "default": "", "required": true, "description": "List of MLLMBatchRequest to process" } ], "return_annotation": "List[int]", "docstring": "Insert requests for batch processing.\n\nArgs:\n requests: List of MLLMBatchRequest to process\n\nReturns:\n List of UIDs assigned to requests", "summary": "Insert requests for batch processing.", "implementation": "Method `MLLMBatchGenerator.insert` updates `self.uid_counter`, `self.unprocessed_requests`; calls `self.unprocessed_requests.append`, `uids.append`, `sorted`, `logger.debug`; returns `uids`.", "documented": true, "public": true, "addressable": true, "line": 816, "end_line": 846, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L816-L846", "decorators": [], "calls": [ "self.unprocessed_requests.append", "uids.append", "sorted", "logger.debug", "len" ], "state_reads": [ "self.uid_counter", "self.unprocessed_requests.append", "self.unprocessed_requests" ], "state_writes": [ "self.uid_counter", "self.unprocessed_requests" ], "raises": [], "return_expressions": [ "uids" ], "awaits": false, "yields": false }, { "name": "remove", "qualname": "MLLMBatchGenerator.remove", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.remove", "kind": "method", "signature": "def remove(self, uids: List[int]) -> None", "parameters": [ { "name": "uids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "List of UIDs to remove" } ], "return_annotation": "None", "docstring": "Remove requests from processing.\n\nArgs:\n uids: List of UIDs to remove", "summary": "Remove requests from processing.", "implementation": "Method `MLLMBatchGenerator.remove` updates `self.active_batch`, `self.unprocessed_requests`; calls `set`, `enumerate`, `self.active_batch.filter`.", "documented": true, "public": true, "addressable": true, "line": 848, "end_line": 870, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L848-L870", "decorators": [], "calls": [ "set", "enumerate", "self.active_batch.filter" ], "state_reads": [ "self.active_batch", "self.active_batch.uids", "self.active_batch.filter", "self.unprocessed_requests" ], "state_writes": [ "self.active_batch", "self.unprocessed_requests" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_preprocess_request", "qualname": "MLLMBatchGenerator._preprocess_request", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._preprocess_request", "kind": "method", "signature": "def _preprocess_request(self, request: MLLMBatchRequest) -> None", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "MLLMBatchRequest", "default": "", "required": true, "description": "Request to preprocess" } ], "return_annotation": "None", "docstring": "Preprocess a single MLLM request (vision encoding).\n\nThis prepares the inputs by:\n1. Processing images/videos through the processor\n2. Tokenizing the prompt with image tokens\n3. Running vision encoder to get features\n\nUses vision cache to skip processing for repeated images.\nIdempotent: if input_ids is already set, returns immediately.\n\nArgs:\n request: Request to preprocess", "summary": "Preprocess a single MLLM request (vision encoding).", "implementation": "Method `MLLMBatchGenerator._preprocess_request` updates `self._stats.num_images_processed`, `self._stats.vision_encoding_time`; calls `time.perf_counter`, `process_image_input`, `all_images.append`, `logger.warning`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 872, "end_line": 1023, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L872-L1023", "decorators": [], "calls": [ "time.perf_counter", "process_image_input", "all_images.append", "logger.warning", "process_video_input", "extract_video_frames_smart", "save_frames_to_temp", "all_images.extend", "process_audio_input", "all_audio.append", "self.vision_cache.get_pixel_cache", "dict", "logger.debug", "getattr", "prepare_inputs", "inputs.get", "inputs.items", "request.extra_kwargs.pop", "self.vision_cache.set_pixel_cache", "len", "bool" ], "state_reads": [ "self.vision_cache.get_pixel_cache", "self.vision_cache", "self.model", "self.processor", "self.vision_cache.set_pixel_cache", "self._stats" ], "state_writes": [ "self._stats.num_images_processed", "self._stats.vision_encoding_time" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_copy_prefix_cache", "qualname": "MLLMBatchGenerator._copy_prefix_cache", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._copy_prefix_cache", "kind": "method", "signature": "def _copy_prefix_cache(cache_list)", "parameters": [ { "name": "cache_list", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Create shallow copies of cache objects to prevent mutation of stored prefix cache.\n\nMLX arrays are immutable and safe to share, but cache objects have mutable\nPython attributes (offset, _idx) that get modified by update_and_fetch().\nWithout copying, the stored prefix cache entry is corrupted after each use.", "summary": "Create shallow copies of cache objects to prevent mutation of stored prefix cache.", "implementation": "Method `MLLMBatchGenerator._copy_prefix_cache` calls `isinstance`, `RotatingKVCache`, `copies.append`, `KVCache`; returns `copies`.", "documented": true, "public": false, "addressable": true, "line": 1026, "end_line": 1054, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1026-L1054", "decorators": [ "staticmethod" ], "calls": [ "isinstance", "RotatingKVCache", "copies.append", "KVCache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "copies" ], "awaits": false, "yields": false }, { "name": "_has_empty_rotating_cache", "qualname": "MLLMBatchGenerator._has_empty_rotating_cache", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._has_empty_rotating_cache", "kind": "method", "signature": "def _has_empty_rotating_cache(cache_list)", "parameters": [ { "name": "cache_list", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Check if any RotatingKVCache layer has no data (keys=None).\n\nThis happens when prefix cache stores a long response where all\nsliding-window entries were trimmed (entries_to_keep=0).\nUsing such a cache produces garbage — fall through to full prefill.", "summary": "Check if any RotatingKVCache layer has no data (keys=None).", "implementation": "Method `MLLMBatchGenerator._has_empty_rotating_cache` calls `isinstance`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1057, "end_line": 1069, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1057-L1069", "decorators": [ "staticmethod" ], "calls": [ "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "_trim_rotating_caches", "qualname": "MLLMBatchGenerator._trim_rotating_caches", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._trim_rotating_caches", "kind": "method", "signature": "def _trim_rotating_caches(cache_list)", "parameters": [ { "name": "cache_list", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Trim RotatingKVCache buffers restored from prefix cache.\n\nPrefix cache stores the full KV state (offset may exceed max_size for\nsliding-window layers). RotatingKVCache._update_in_place computes\n``new_size = min(step, max_size - prev)`` which goes negative when\n``prev > max_size``, crashing with \"Negative dimensions not allowed\".\n\nTrimming the buffer to max_size and clamping offset/idx prevents this.", "summary": "Trim RotatingKVCache buffers restored from prefix cache.", "implementation": "Method `MLLMBatchGenerator._trim_rotating_caches` calls `isinstance`, `layer_cache._trim`, `min`, `logger.warning`.", "documented": true, "public": false, "addressable": true, "line": 1072, "end_line": 1106, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1072-L1106", "decorators": [ "staticmethod" ], "calls": [ "isinstance", "layer_cache._trim", "min", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_run_chunked_text_prefill", "qualname": "MLLMBatchGenerator._run_chunked_text_prefill", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._run_chunked_text_prefill", "kind": "method", "signature": "def _run_chunked_text_prefill(self, request: MLLMBatchRequest, cache: List[Any]) -> mx.array", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "MLLMBatchRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Run prefill in chunks for text-only requests, reporting real progress.\n\nProcesses input_ids in prefill_step_size chunks through the language\nmodel, updating ``_prefill_progress`` after each chunk so the status\nendpoint can report accurate prefill percentage.\n\nReturns:\n Logits from the last chunk (same contract as _run_vision_encoding).", "summary": "Run prefill in chunks for text-only requests, reporting real progress.", "implementation": "Method `MLLMBatchGenerator._run_chunked_text_prefill` calls `self.language_model`, `request.extra_kwargs.clear`, `hasattr`, `logger.info`; can raise `PrefillAbortedError`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1108, "end_line": 1206, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1108-L1206", "decorators": [], "calls": [ "self.language_model", "request.extra_kwargs.clear", "hasattr", "logger.info", "self._aborted_request_ids.discard", "PrefillAbortedError", "_eval_prompt_cache", "mx.clear_cache" ], "state_reads": [ "self.prefill_step_size", "self._prefill_progress", "self.language_model", "self._aborted_request_ids", "self._aborted_request_ids.discard" ], "state_writes": [], "raises": [ "PrefillAbortedError" ], "return_expressions": [ "output.logits", "output" ], "awaits": false, "yields": false }, { "name": "_run_vision_encoding", "qualname": "MLLMBatchGenerator._run_vision_encoding", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._run_vision_encoding", "kind": "method", "signature": "def _run_vision_encoding(self, request: MLLMBatchRequest, cache: Optional[List[Any]]=None) -> mx.array", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "MLLMBatchRequest", "default": "", "required": true, "description": "Preprocessed request with input_ids and pixel_values" }, { "name": "cache", "kind": "positional or keyword", "annotation": "Optional[List[Any]]", "default": "None", "required": false, "description": "KV cache list for the language model. If provided, the language model writes its KV state directly into this cache during the forward pass." } ], "return_annotation": "mx.array", "docstring": "Run the initial VLM forward pass to encode vision and get first logits.\n\nThis runs the full VLM model (vision + language) on the prompt,\nwhich encodes the images and fills the provided KV cache.\n\nArgs:\n request: Preprocessed request with input_ids and pixel_values\n cache: KV cache list for the language model. If provided, the\n language model writes its KV state directly into this cache\n during the forward pass.\n\nReturns:\n Logits from the forward pass", "summary": "Run the initial VLM forward pass to encode vision and get first logits.", "implementation": "Method `MLLMBatchGenerator._run_vision_encoding` calls `dict`, `self.model`, `request.extra_kwargs.clear`, `hasattr`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1208, "end_line": 1258, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1208-L1258", "decorators": [], "calls": [ "dict", "self.model", "request.extra_kwargs.clear", "hasattr" ], "state_reads": [ "self.model" ], "state_writes": [], "raises": [], "return_expressions": [ "output.logits", "output" ], "awaits": false, "yields": false }, { "name": "_process_prompts", "qualname": "MLLMBatchGenerator._process_prompts", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._process_prompts", "kind": "method", "signature": "def _process_prompts(self, requests: List[MLLMBatchRequest]) -> MLLMBatch", "parameters": [ { "name": "requests", "kind": "positional or keyword", "annotation": "List[MLLMBatchRequest]", "default": "", "required": true, "description": "Requests to process" } ], "return_annotation": "MLLMBatch", "docstring": "Process a batch of requests through vision encoding and initial prefill.\n\nFor MLLM, this is more complex than LLM:\n1. Preprocess each request (tokenize, process images)\n2. Run vision encoding per-request with individual KVCache objects\n3. Merge individual caches into a BatchKVCache for generation\n\nArgs:\n requests: Requests to process\n\nReturns:\n MLLMBatch ready for generation", "summary": "Process a batch of requests through vision encoding and initial prefill.", "implementation": "Method `MLLMBatchGenerator._process_prompts` updates `self._stats.prompt_tokens`, `self._stats.prompt_time`; calls `time.perf_counter`, `self._preprocess_request`, `logger.error`, `type`; can raise `PrefillAbortedError`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1260, "end_line": 1682, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1260-L1682", "decorators": [], "calls": [ "time.perf_counter", "self._preprocess_request", "logger.error", "type", "failed_requests.append", "requests.remove", "self._pending_error_responses.append", "MLLMBatchResponse", "mx.zeros", "combined.extend", "make_logits_processors", "logger.info", "len", "make_sampler", "sum", "logger.warning", "self._aborted_request_ids.discard", "PrefillAbortedError", "req.input_ids.reshape(-1).tolist", "req.input_ids.reshape", "self.prefix_cache.fetch", "list", "getattr", "self._has_empty_rotating_cache", "self._copy_prefix_cache", "self._trim_rotating_caches", "mx.array", "mx.stream", "self.language_model", "_eval_prompt_cache", "mx.clear_cache", "hasattr", "_sample_first_token", "first_tokens.append", "sampled.item", "all_logprobs.append", "logprobs.squeeze", "per_request_caches.append", "logger.debug", "_trim_cache_offset", "make_prompt_cache", "self._run_chunked_text_prefill", "self._run_vision_encoding", "aborted_requests.append", "self._prefill_progress.pop", "isinstance", "layer_cache._temporal_order", "per_request_caches[0][layer_idx].merge", "range", "logits_processors_by_request.get", "any", "samplers_by_request.get", "req.extra_kwargs.clear", "MLLMBatch" ], "state_reads": [ "self._preprocess_request", "self._pending_error_responses.append", "self._pending_error_responses", "self._stats", "self.prefill_step_size", "self._aborted_request_ids", "self._aborted_request_ids.discard", "self.prefix_cache", "self._think_suffix_len", "self.prefix_cache.fetch", "self.model", "self._has_empty_rotating_cache", "self._copy_prefix_cache", "self._trim_rotating_caches", "self._prefill_progress", "self.language_model", "self.max_kv_size", "self._run_chunked_text_prefill", "self._run_vision_encoding", "self._prefill_progress.pop" ], "state_writes": [ "self._stats.prompt_tokens", "self._stats.prompt_time" ], "raises": [ "PrefillAbortedError" ], "return_expressions": [ "None", "MLLMBatch(uids=[req.uid for req in requests], request_ids=[req.request_id for req in requests], y=y, logprobs=all_logpr…" ], "awaits": false, "yields": false }, { "name": "_sample_first_token", "qualname": "MLLMBatchGenerator._process_prompts._sample_first_token", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._process_prompts._sample_first_token", "kind": "nested function", "signature": "def _sample_first_token(req: MLLMBatchRequest, logits: mx.array)", "parameters": [ { "name": "req", "kind": "positional or keyword", "annotation": "MLLMBatchRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `MLLMBatchGenerator._process_prompts._sample_first_token` calls `logits_processors_by_request.get`, `mx.array`, `processor`, `mx.logsumexp`; returns `(sampled, logprobs)`.", "implementation": "Nested Function `MLLMBatchGenerator._process_prompts._sample_first_token` calls `logits_processors_by_request.get`, `mx.array`, `processor`, `mx.logsumexp`; returns `(sampled, logprobs)`.", "documented": false, "public": false, "addressable": false, "line": 1348, "end_line": 1362, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1348-L1362", "decorators": [], "calls": [ "logits_processors_by_request.get", "mx.array", "processor", "mx.logsumexp", "samplers_by_request.get", "sampler", "mx.eval" ], "state_reads": [ "self.sampler" ], "state_writes": [], "raises": [], "return_expressions": [ "(sampled, logprobs)" ], "awaits": false, "yields": false }, { "name": "_step", "qualname": "MLLMBatchGenerator._step", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._step", "kind": "method", "signature": "def _step(self, input_tokens: mx.array, cache: List[Any], logits_processors: Optional[List[Optional[List[Callable]]]]=None, output_tokens: Optional[List[List[int]]]=None, samplers: Optional[List[Optional[Callable]]]=None) -> Tuple[mx.array, List[mx.array]]", "parameters": [ { "name": "input_tokens", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Input tokens [batch_size, 1] or [batch_size]" }, { "name": "cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "BatchKVCache for the language model" }, { "name": "logits_processors", "kind": "positional or keyword", "annotation": "Optional[List[Optional[List[Callable]]]]", "default": "None", "required": false, "description": "Per-request logits processors (e.g. repetition penalty)" }, { "name": "output_tokens", "kind": "positional or keyword", "annotation": "Optional[List[List[int]]]", "default": "None", "required": false, "description": "Per-request generated tokens so far (needed by processors)" }, { "name": "samplers", "kind": "positional or keyword", "annotation": "Optional[List[Optional[Callable]]]", "default": "None", "required": false, "description": "Per-request sampler functions (for top_k/min_p)" } ], "return_annotation": "Tuple[mx.array, List[mx.array]]", "docstring": "Run one generation step through the language model.\n\nArgs:\n input_tokens: Input tokens [batch_size, 1] or [batch_size]\n cache: BatchKVCache for the language model\n logits_processors: Per-request logits processors (e.g. repetition penalty)\n output_tokens: Per-request generated tokens so far (needed by processors)\n samplers: Per-request sampler functions (for top_k/min_p)\n\nReturns:\n Tuple of (sampled tokens, logprobs list)", "summary": "Run one generation step through the language model.", "implementation": "Method `MLLMBatchGenerator._step` calls `self.language_model`, `hasattr`, `any`, `range`; returns `(sampled, list(logprobs))`.", "documented": true, "public": false, "addressable": true, "line": 1684, "end_line": 1746, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1684-L1746", "decorators": [], "calls": [ "self.language_model", "hasattr", "any", "range", "processor", "mx.array", "processed_logits.append", "mx.concatenate", "mx.logsumexp", "sampled_list.append", "s", "self.sampler", "list" ], "state_reads": [ "self.language_model", "self.sampler" ], "state_writes": [], "raises": [], "return_expressions": [ "(sampled, list(logprobs))" ], "awaits": false, "yields": false }, { "name": "_next", "qualname": "MLLMBatchGenerator._next", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._next", "kind": "method", "signature": "def _next(self) -> List[MLLMBatchResponse]", "parameters": [], "return_annotation": "List[MLLMBatchResponse]", "docstring": "Internal next() implementation.\n\nReturns:\n List of MLLMBatchResponse for this step", "summary": "Internal next() implementation.", "implementation": "Method `MLLMBatchGenerator._next` updates `self.active_batch`, `self.unprocessed_requests`, `self._stats.prompt_time`, `self._stats.generation_time`; calls `time.perf_counter`, `len`, `self._process_prompts`, `logger.error`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1748, "end_line": 1964, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1748-L1964", "decorators": [], "calls": [ "time.perf_counter", "len", "self._process_prompts", "logger.error", "type", "self._pending_error_responses.append", "MLLMBatchResponse", "mx.zeros", "batch.extend", "logger.warning", "list", "self._pending_error_responses.clear", "y.tolist", "zip", "self._step", "mx.async_eval", "enumerate", "req.output_tokens.append", "_processors_can_retire", "_drop_retired_processors", "logger.info", "end_idx.append", "keep_idx.append", "self._prefill_progress.pop", "responses.append", "self._maybe_store_prefix_cache", "batch.filter" ], "state_reads": [ "self.active_batch", "self.unprocessed_requests", "self.completion_batch_size", "self._process_prompts", "self._pending_error_responses.append", "self._pending_error_responses", "self._pending_error_responses.clear", "self._step", "self._stats", "self.stop_tokens", "self._prefill_progress.pop", "self._prefill_progress", "self._maybe_store_prefix_cache" ], "state_writes": [ "self.active_batch", "self.unprocessed_requests", "self._stats.prompt_time", "self._stats.generation_time", "self._stats.generation_tokens" ], "raises": [], "return_expressions": [ "[]", "error_responses", "error_responses + responses" ], "awaits": false, "yields": false }, { "name": "next", "qualname": "MLLMBatchGenerator.next", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.next", "kind": "method", "signature": "def next(self) -> List[MLLMBatchResponse]", "parameters": [], "return_annotation": "List[MLLMBatchResponse]", "docstring": "Generate next token for all requests in the batch.\n\nReturns:\n List of MLLMBatchResponse, one per active request", "summary": "Generate next token for all requests in the batch.", "implementation": "Method `MLLMBatchGenerator.next` calls `mx.stream`, `self._next`; returns `self._next()`.", "documented": true, "public": true, "addressable": true, "line": 1966, "end_line": 1974, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1966-L1974", "decorators": [], "calls": [ "mx.stream", "self._next" ], "state_reads": [ "self._next" ], "state_writes": [], "raises": [], "return_expressions": [ "self._next()" ], "awaits": false, "yields": false }, { "name": "stats", "qualname": "MLLMBatchGenerator.stats", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.stats", "kind": "method", "signature": "def stats(self) -> MLLMBatchStats", "parameters": [], "return_annotation": "MLLMBatchStats", "docstring": "Get generation statistics.\n\nReturns:\n MLLMBatchStats with timing and token counts", "summary": "Get generation statistics.", "implementation": "Method `MLLMBatchGenerator.stats` updates `self._stats.peak_memory`; calls `mx.get_peak_memory`; returns `self._stats`.", "documented": true, "public": true, "addressable": true, "line": 1976, "end_line": 1984, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1976-L1984", "decorators": [], "calls": [ "mx.get_peak_memory" ], "state_reads": [ "self._stats" ], "state_writes": [ "self._stats.peak_memory" ], "raises": [], "return_expressions": [ "self._stats" ], "awaits": false, "yields": false }, { "name": "_maybe_store_prefix_cache", "qualname": "MLLMBatchGenerator._maybe_store_prefix_cache", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator._maybe_store_prefix_cache", "kind": "method", "signature": "def _maybe_store_prefix_cache(self, batch: MLLMBatch, end_indices: List[int]) -> None", "parameters": [ { "name": "batch", "kind": "positional or keyword", "annotation": "MLLMBatch", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "end_indices", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Store KV caches for finished text-only requests into prefix cache.\n\nMust be called BEFORE batch.filter() so that indices are still valid.", "summary": "Store KV caches for finished text-only requests into prefix cache.", "implementation": "Method `MLLMBatchGenerator._maybe_store_prefix_cache` calls `batch.extract_cache`, `req.input_ids.reshape(-1).tolist`, `req.input_ids.reshape`, `_trim_cache_offset`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1986, "end_line": 2014, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L1986-L2014", "decorators": [], "calls": [ "batch.extract_cache", "req.input_ids.reshape(-1).tolist", "req.input_ids.reshape", "_trim_cache_offset", "self.prefix_cache.store", "logger.warning", "type" ], "state_reads": [ "self.prefix_cache", "self._think_suffix_len", "self.prefix_cache.store" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_prefill_progress", "qualname": "MLLMBatchGenerator.get_prefill_progress", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.get_prefill_progress", "kind": "method", "signature": "def get_prefill_progress(self, request_id: str) -> Optional[Tuple[int, int]]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[Tuple[int, int]]", "docstring": "Return (processed_tokens, total_tokens) or None.", "summary": "Return (processed_tokens, total_tokens) or None.", "implementation": "Method `MLLMBatchGenerator.get_prefill_progress` calls `self._prefill_progress.get`; returns `self._prefill_progress.get(request_id)`.", "documented": true, "public": true, "addressable": true, "line": 2016, "end_line": 2018, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2016-L2018", "decorators": [], "calls": [ "self._prefill_progress.get" ], "state_reads": [ "self._prefill_progress.get", "self._prefill_progress" ], "state_writes": [], "raises": [], "return_expressions": [ "self._prefill_progress.get(request_id)" ], "awaits": false, "yields": false }, { "name": "get_vision_cache_stats", "qualname": "MLLMBatchGenerator.get_vision_cache_stats", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.get_vision_cache_stats", "kind": "method", "signature": "def get_vision_cache_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get vision cache statistics.", "summary": "Get vision cache statistics.", "implementation": "Method `MLLMBatchGenerator.get_vision_cache_stats` calls `self.vision_cache.get_stats`; returns `self.vision_cache.get_stats()`.", "documented": true, "public": true, "addressable": true, "line": 2020, "end_line": 2022, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2020-L2022", "decorators": [], "calls": [ "self.vision_cache.get_stats" ], "state_reads": [ "self.vision_cache.get_stats", "self.vision_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "self.vision_cache.get_stats()" ], "awaits": false, "yields": false }, { "name": "get_prefix_cache_stats", "qualname": "MLLMBatchGenerator.get_prefix_cache_stats", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.get_prefix_cache_stats", "kind": "method", "signature": "def get_prefix_cache_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get KV prefix cache statistics.", "summary": "Get KV prefix cache statistics.", "implementation": "Method `MLLMBatchGenerator.get_prefix_cache_stats` calls `self.prefix_cache.get_stats`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 2024, "end_line": 2038, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2024-L2038", "decorators": [], "calls": [ "self.prefix_cache.get_stats" ], "state_reads": [ "self.prefix_cache", "self.prefix_cache.get_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "self.prefix_cache.get_stats()", "{'hits': 0, 'misses': 0, 'hit_rate': 0.0, 'evictions': 0, 'tokens_saved': 0, 'current_memory_mb': 0.0, 'max_memory_mb':…" ], "awaits": false, "yields": false }, { "name": "has_pending", "qualname": "MLLMBatchGenerator.has_pending", "full_name": "vllm_mlx.mllm_batch_generator.MLLMBatchGenerator.has_pending", "kind": "method", "signature": "def has_pending(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if there are pending or active requests.", "summary": "Check if there are pending or active requests.", "implementation": "Method `MLLMBatchGenerator.has_pending` calls `bool`; returns `bool(self.unprocessed_requests or self.active_batch)`.", "documented": true, "public": true, "addressable": true, "line": 2040, "end_line": 2042, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2040-L2042", "decorators": [], "calls": [ "bool" ], "state_reads": [ "self.unprocessed_requests", "self.active_batch" ], "state_writes": [], "raises": [], "return_expressions": [ "bool(self.unprocessed_requests or self.active_batch)" ], "awaits": false, "yields": false }, { "name": "install_mtp_mllm", "qualname": "install_mtp_mllm", "full_name": "vllm_mlx.mllm_batch_generator.install_mtp_mllm", "kind": "function", "signature": "def install_mtp_mllm(batch_gen: 'MLLMBatchGenerator', language_model: Any, num_draft_tokens: int=1) -> None", "parameters": [ { "name": "batch_gen", "kind": "positional or keyword", "annotation": "'MLLMBatchGenerator'", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "language_model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_draft_tokens", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Optional positional or keyword input; defaults to `1`." } ], "return_annotation": "None", "docstring": "Install MTP (Multi-Token Prediction) on an MLLMBatchGenerator.\n\nAdapts the always-advance MTP strategy from scheduler._install_mtp\nfor the MLLM batched generation path. Handles hybrid model caches\n(BatchKVCache for attention + ArraysCache for recurrent layers).\n\nFlow per generation step:\n1. Use skip_state logits/hidden OR run model forward -> sample primary\n2. MTP head drafts one token\n3. Verify [primary, draft] in one model call (always advances cache)\n4. Accept: skip_state from pos 1, defer draft for next step emission\n Reject: trim KV by 2 + restore RNN state + re-advance with primary\n5. Draft is emitted in the NEXT generation step after primary", "summary": "Install MTP (Multi-Token Prediction) on an MLLMBatchGenerator.", "implementation": "Function `install_mtp_mllm` calls `make_sampler`, `threading.Lock`, `logger.warning`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 2045, "end_line": 2590, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2045-L2590", "decorators": [], "calls": [ "make_sampler", "threading.Lock", "logger.warning", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_get_mtp_stats", "qualname": "install_mtp_mllm._get_mtp_stats", "full_name": "vllm_mlx.mllm_batch_generator.install_mtp_mllm._get_mtp_stats", "kind": "nested function", "signature": "def _get_mtp_stats() -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "", "summary": "Nested Function `install_mtp_mllm._get_mtp_stats` calls `dict`; returns `{'enabled': True, 'requested_draft_tokens': num_draft_tokens, 'effective_draft_tokens': 1, 'mode': 'request_local_sampl…`.", "implementation": "Nested Function `install_mtp_mllm._get_mtp_stats` calls `dict`; returns `{'enabled': True, 'requested_draft_tokens': num_draft_tokens, 'effective_draft_tokens': 1, 'mode': 'request_local_sampl…`.", "documented": false, "public": false, "addressable": false, "line": 2089, "end_line": 2110, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2089-L2110", "decorators": [], "calls": [ "dict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'enabled': True, 'requested_draft_tokens': num_draft_tokens, 'effective_draft_tokens': 1, 'mode': 'request_local_sampl…" ], "awaits": false, "yields": false }, { "name": "_mtp_step", "qualname": "install_mtp_mllm._mtp_step", "full_name": "vllm_mlx.mllm_batch_generator.install_mtp_mllm._mtp_step", "kind": "nested function", "signature": "def _mtp_step(input_tokens: mx.array, cache: List[Any], logits_processors: Optional[List[Optional[List[Callable]]]]=None, output_tokens: Optional[List[List[int]]]=None, samplers: Optional[List[Optional[Callable]]]=None) -> Tuple[mx.array, List[mx.array]]", "parameters": [ { "name": "input_tokens", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits_processors", "kind": "positional or keyword", "annotation": "Optional[List[Optional[List[Callable]]]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "output_tokens", "kind": "positional or keyword", "annotation": "Optional[List[List[int]]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "samplers", "kind": "positional or keyword", "annotation": "Optional[List[Optional[Callable]]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "Tuple[mx.array, List[mx.array]]", "docstring": "Extended _step with MTP always-advance strategy.", "summary": "Extended _step with MTP always-advance strategy.", "implementation": "Nested Function `install_mtp_mllm._mtp_step` calls `list`, `any`, `_skip_state_by_uid.clear`, `_orig_step`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 2114, "end_line": 2455, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2114-L2455", "decorators": [], "calls": [ "list", "any", "_skip_state_by_uid.clear", "_orig_step", "_skip_state_by_uid.pop", "all", "logger.debug", "mx.concatenate", "language_model", "isinstance", "range", "processor", "mx.array", "processed_logits.append", "mx.logsumexp", "sampled_list.append", "s", "batch_gen.sampler", "language_model.mtp_forward", "_request_uses_stochastic_sampling", "_sampling_logprobs", "enumerate", "mx.random.categorical", "_draft_sampler", "hasattr", "_c.is_trimmable", "draft_tokens.tolist", "mx.random.uniform", "mx.eval", "int", "_accept_sampled_draft", "float", "verify_distribution[row, draft_token].item", "draft_distribution[row, draft_token].item", "draws[row].item", "_residual_logprobs", "residual_token.item", "mx.argmax", "verify_pred.tolist", "mx.async_eval", "bool", "c.is_trimmable", "c.trim", "_rnn_snapshots.items", "_deferred_drafts.pop", "logger.warning", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_orig_step(input_tokens, cache, logits_processors, output_tokens, samplers)", "(primary_tokens, list(logprobs))" ], "awaits": false, "yields": false }, { "name": "_mtp_next", "qualname": "install_mtp_mllm._mtp_next", "full_name": "vllm_mlx.mllm_batch_generator.install_mtp_mllm._mtp_next", "kind": "nested function", "signature": "def _mtp_next() -> List[MLLMBatchResponse]", "parameters": [], "return_annotation": "List[MLLMBatchResponse]", "docstring": "Wrapper around _next that emits deferred MTP draft tokens.", "summary": "Wrapper around _next that emits deferred MTP draft tokens.", "implementation": "Nested Function `install_mtp_mllm._mtp_next` calls `_skip_state_by_uid.clear`, `_deferred_drafts.clear`, `_attempted_drafts_by_uid.clear`, `_deferred_drafts.pop`; returns `augmented`.", "documented": true, "public": false, "addressable": false, "line": 2460, "end_line": 2576, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2460-L2576", "decorators": [], "calls": [ "_skip_state_by_uid.clear", "_deferred_drafts.clear", "_attempted_drafts_by_uid.clear", "_deferred_drafts.pop", "batch_gen._inner_next", "_mark_mtp_attempts_on_primary_responses", "set", "augmented.append", "_skip_state_by_uid.pop", "prev_deferred.pop", "MLLMBatchResponse", "draft_end_uids.add", "enumerate", "batch.requests[e].output_tokens.append", "batch_gen._maybe_store_prefix_cache", "batch_gen.active_batch.filter", "list" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "augmented" ], "awaits": false, "yields": false }, { "name": "install_chunked_prefill_mllm", "qualname": "install_chunked_prefill_mllm", "full_name": "vllm_mlx.mllm_batch_generator.install_chunked_prefill_mllm", "kind": "function", "signature": "def install_chunked_prefill_mllm(batch_gen: 'MLLMBatchGenerator', budget: int=1024) -> None", "parameters": [ { "name": "batch_gen", "kind": "positional or keyword", "annotation": "'MLLMBatchGenerator'", "default": "", "required": true, "description": "The MLLMBatchGenerator to patch." }, { "name": "budget", "kind": "positional or keyword", "annotation": "int", "default": "1024", "required": false, "description": "Max tokens to prefill per step (chunk size)." } ], "return_annotation": "None", "docstring": "Install interleaved prefill/decode on an MLLMBatchGenerator.\n\nWhen a long text-only request arrives, instead of blocking the entire\nevent loop for 20-60+ seconds during prefill, this processes ONE chunk\nof the new request's prefill per ``step()`` call. Between steps the\nscheduler yields to the event loop (``await asyncio.sleep(0)``), so\nhealth/status/metrics endpoints remain responsive.\n\nWhen an active batch is generating, prefill chunks are interleaved with\ngeneration steps to keep throughput for existing requests at 30-50 tok/s.\n\nArgs:\n batch_gen: The MLLMBatchGenerator to patch.\n budget: Max tokens to prefill per step (chunk size).", "summary": "Install interleaved prefill/decode on an MLLMBatchGenerator.", "implementation": "Function `install_chunked_prefill_mllm` calls `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 2593, "end_line": 3073, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2593-L3073", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_generation_step", "qualname": "install_chunked_prefill_mllm._generation_step", "full_name": "vllm_mlx.mllm_batch_generator.install_chunked_prefill_mllm._generation_step", "kind": "nested function", "signature": "def _generation_step() -> List[MLLMBatchResponse]", "parameters": [], "return_annotation": "List[MLLMBatchResponse]", "docstring": "Run one generation step for the active batch. Returns responses.", "summary": "Run one generation step for the active batch.", "implementation": "Nested Function `install_chunked_prefill_mllm._generation_step` calls `list`, `batch_gen._pending_error_responses.clear`, `time.perf_counter`, `batch_gen._step`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 2623, "end_line": 2713, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2623-L2713", "decorators": [], "calls": [ "list", "batch_gen._pending_error_responses.clear", "time.perf_counter", "batch_gen._step", "mx.eval", "y.tolist", "enumerate", "zip", "req.output_tokens.append", "end_idx.append", "keep_idx.append", "batch_gen._prefill_progress.pop", "responses.append", "MLLMBatchResponse", "batch_gen._maybe_store_prefix_cache", "batch.filter", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "error_responses", "error_responses + responses" ], "awaits": false, "yields": false }, { "name": "_chunked_next", "qualname": "install_chunked_prefill_mllm._chunked_next", "full_name": "vllm_mlx.mllm_batch_generator.install_chunked_prefill_mllm._chunked_next", "kind": "nested function", "signature": "def _chunked_next() -> List[MLLMBatchResponse]", "parameters": [], "return_annotation": "List[MLLMBatchResponse]", "docstring": "Interleaved prefill/decode: one prefill chunk + one gen step.", "summary": "Interleaved prefill/decode: one prefill chunk + one gen step.", "implementation": "Nested Function `install_chunked_prefill_mllm._chunked_next` calls `batch_gen._aborted_request_ids.discard`, `mx.clear_cache`, `batch_gen._prefill_progress.pop`, `batch_gen._pending_error_responses.append`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 2715, "end_line": 3058, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L2715-L3058", "decorators": [], "calls": [ "batch_gen._aborted_request_ids.discard", "mx.clear_cache", "batch_gen._prefill_progress.pop", "batch_gen._pending_error_responses.append", "MLLMBatchResponse", "mx.zeros", "_generation_step", "time.perf_counter", "batch_gen.language_model", "_eval_prompt_cache", "batch_gen._preprocess_request", "short_reqs.append", "batch_gen._process_prompts", "batch_gen.active_batch.extend", "logger.warning", "hasattr", "getattr", "mx.array", "processor", "mx.logsumexp", "batch_gen.sampler", "mx.eval", "req_lp.extend", "make_logits_processors", "make_sampler", "MLLMBatch", "logprobs.squeeze", "batch_gen._trim_rotating_caches", "isinstance", "layer_cache._temporal_order", "request_cache[layer_idx].merge", "range", "len", "req.input_ids.reshape(-1).tolist", "req.input_ids.reshape", "_trim_cache_offset", "batch_gen.prefix_cache.store", "logger.info", "logger.error", "batch_gen.unprocessed_requests.remove", "input_ids.reshape(-1).tolist", "input_ids.reshape", "batch_gen.prefix_cache.fetch", "list", "batch_gen._has_empty_rotating_cache", "batch_gen._copy_prefix_cache", "make_prompt_cache", "_orig_next" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_generation_step()", "[]", "_orig_next()" ], "awaits": false, "yields": false }, { "name": "_patched_remove", "qualname": "install_chunked_prefill_mllm._patched_remove", "full_name": "vllm_mlx.mllm_batch_generator.install_chunked_prefill_mllm._patched_remove", "kind": "nested function", "signature": "def _patched_remove(uids: List[int]) -> None", "parameters": [ { "name": "uids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `install_chunked_prefill_mllm._patched_remove` calls `set`, `mx.clear_cache`, `_orig_remove`.", "implementation": "Nested Function `install_chunked_prefill_mllm._patched_remove` calls `set`, `mx.clear_cache`, `_orig_remove`.", "documented": false, "public": false, "addressable": false, "line": 3063, "end_line": 3068, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_batch_generator.py#L3063-L3068", "decorators": [], "calls": [ "set", "mx.clear_cache", "_orig_remove" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.mllm_cache", "path": "vllm_mlx/mllm_cache.py", "page_path": "reference/api/vllm_mlx/mllm_cache.md", "docstring": "MLLM (Multimodal Language Model) Prefix Cache Manager.\n\nThis module provides advanced caching for MLLM inference, implementing\nthe LMCache-style approach for multimodal prefix caching:\n\nFeatures:\n- Image content hashing for cache keys (LMCache style)\n- Vision embedding caching (skip encoder on hit)\n- KV cache state caching with prefix matching\n- Token ID tracking for partial prefix reuse\n- LRU eviction policy with memory limits\n- Stats tracking (hits, misses, tokens saved, encoder skips)\n\nBased on research from:\n- LMCache: https://blog.lmcache.ai/2025-07-03-multimodal-models/\n- vLLM Prefix Caching: https://docs.vllm.ai/en/stable/design/prefix_caching/\n- mlx-lm cache_prompt: https://github.com/ml-explore/mlx-lm", "summary": "MLLM (Multimodal Language Model) Prefix Cache Manager.", "line_count": 459, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L1-L459", "members": [ "logger", "MLLMCacheStats", "MLLMPrefixCacheEntry", "compute_image_hash", "compute_images_hash", "MLLMPrefixCacheManager", "MLLMCacheManager", "VLMCacheStats", "VLMPrefixCacheEntry", "VLMCacheEntry", "VLMPrefixCacheManager", "VLMCacheManager" ], "symbols": [ { "name": "MLLMCacheStats", "qualname": "MLLMCacheStats", "full_name": "vllm_mlx.mllm_cache.MLLMCacheStats", "kind": "class", "signature": "class MLLMCacheStats", "parameters": [ { "name": "hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "partial_hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "tokens_saved", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "image_cache_hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "vision_encoder_skips", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "total_queries", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "evictions", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "MLLMCacheStats", "docstring": "Statistics for MLLM cache performance.", "summary": "Statistics for MLLM cache performance.", "implementation": "Class `MLLMCacheStats` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L34-L65", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "hit_rate", "qualname": "MLLMCacheStats.hit_rate", "full_name": "vllm_mlx.mllm_cache.MLLMCacheStats.hit_rate", "kind": "method", "signature": "def hit_rate(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Calculate cache hit rate.", "summary": "Calculate cache hit rate.", "implementation": "Method `MLLMCacheStats.hit_rate` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 47, "end_line": 51, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L47-L51", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.total_queries", "self.hits" ], "state_writes": [], "raises": [], "return_expressions": [ "0.0", "self.hits / self.total_queries" ], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "MLLMCacheStats.to_dict", "full_name": "vllm_mlx.mllm_cache.MLLMCacheStats.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Convert stats to dictionary.", "summary": "Convert stats to dictionary.", "implementation": "Method `MLLMCacheStats.to_dict` returns `{'hits': self.hits, 'misses': self.misses, 'partial_hits': self.partial_hits, 'hit_rate': self.hit_rate, 'tokens_saved'…`.", "documented": true, "public": true, "addressable": true, "line": 53, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L53-L65", "decorators": [], "calls": [], "state_reads": [ "self.hits", "self.misses", "self.partial_hits", "self.hit_rate", "self.tokens_saved", "self.image_cache_hits", "self.vision_encoder_skips", "self.total_queries", "self.evictions" ], "state_writes": [], "raises": [], "return_expressions": [ "{'hits': self.hits, 'misses': self.misses, 'partial_hits': self.partial_hits, 'hit_rate': self.hit_rate, 'tokens_saved'…" ], "awaits": false, "yields": false }, { "name": "MLLMPrefixCacheEntry", "qualname": "MLLMPrefixCacheEntry", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheEntry", "kind": "class", "signature": "class MLLMPrefixCacheEntry", "parameters": [ { "name": "image_hash", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt_hash", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "vision_embeddings", "kind": "field", "annotation": "Any", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "kv_cache", "kind": "field", "annotation": "list[Any]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "token_ids", "kind": "field", "annotation": "list[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "num_image_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "num_text_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "created_at", "kind": "field", "annotation": "float", "default": "field(default_factory=time.time)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=time.time)`." }, { "name": "hit_count", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "model_name", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." } ], "return_annotation": "MLLMPrefixCacheEntry", "docstring": "Enhanced cache entry storing vision embeddings, KV cache, and token IDs.\n\nThis enables:\n1. Skipping vision encoder on image cache hit (saves ~1-2s per image)\n2. Skipping prefix computation on token match (saves ~0.5s per 1k tokens)\n3. Partial prefix reuse for multi-turn conversations", "summary": "Enhanced cache entry storing vision embeddings, KV cache, and token IDs.", "implementation": "Class `MLLMPrefixCacheEntry` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L69-L133", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "total_tokens", "qualname": "MLLMPrefixCacheEntry.total_tokens", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheEntry.total_tokens", "kind": "method", "signature": "def total_tokens(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return the number of token IDs represented by this cache entry.", "summary": "Return the number of token IDs represented by this cache entry.", "implementation": "Method `MLLMPrefixCacheEntry.total_tokens` calls `len`; returns `len(self.token_ids)`.", "documented": true, "public": true, "addressable": true, "line": 99, "end_line": 102, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L99-L102", "decorators": [ "property" ], "calls": [ "len" ], "state_reads": [ "self.token_ids" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.token_ids)" ], "awaits": false, "yields": false }, { "name": "memory_size", "qualname": "MLLMPrefixCacheEntry.memory_size", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheEntry.memory_size", "kind": "method", "signature": "def memory_size(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Estimate memory usage in bytes.", "summary": "Estimate memory usage in bytes.", "implementation": "Method `MLLMPrefixCacheEntry.memory_size` calls `hasattr`; returns `size`.", "documented": true, "public": true, "addressable": true, "line": 105, "end_line": 119, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L105-L119", "decorators": [ "property" ], "calls": [ "hasattr" ], "state_reads": [ "self.vision_embeddings", "self.vision_embeddings.nbytes", "self.kv_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "size" ], "awaits": false, "yields": false }, { "name": "get_prefix_match_length", "qualname": "MLLMPrefixCacheEntry.get_prefix_match_length", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheEntry.get_prefix_match_length", "kind": "method", "signature": "def get_prefix_match_length(self, new_token_ids: list[int]) -> int", "parameters": [ { "name": "new_token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Find how many tokens match between cached prefix and new input.\n\nThis is the key to prefix caching - if the first N tokens match,\nwe can skip computing KV states for those N tokens.", "summary": "Find how many tokens match between cached prefix and new input.", "implementation": "Method `MLLMPrefixCacheEntry.get_prefix_match_length` calls `enumerate`, `zip`; returns `match_length`.", "documented": true, "public": true, "addressable": true, "line": 121, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L121-L133", "decorators": [], "calls": [ "enumerate", "zip" ], "state_reads": [ "self.token_ids" ], "state_writes": [], "raises": [], "return_expressions": [ "match_length" ], "awaits": false, "yields": false }, { "name": "compute_image_hash", "qualname": "compute_image_hash", "full_name": "vllm_mlx.mllm_cache.compute_image_hash", "kind": "function", "signature": "def compute_image_hash(image_path: str) -> str", "parameters": [ { "name": "image_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Path to image file" } ], "return_annotation": "str", "docstring": "Compute hash of image content for cache key.\n\nFollowing LMCache approach: hash the actual image bytes, not the path.\nThis ensures cache hits even when the same image is loaded from\ndifferent paths or as base64.\n\nArgs:\n image_path: Path to image file\n\nReturns:\n SHA256 hash of image content (first 16 chars)", "summary": "Compute hash of image content for cache key.", "implementation": "Function `compute_image_hash` calls `Path`, `path.exists`, `path.read_bytes`, `hashlib.sha256(content).hexdigest`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 136, "end_line": 161, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L136-L161", "decorators": [], "calls": [ "Path", "path.exists", "path.read_bytes", "hashlib.sha256(content).hexdigest", "hashlib.sha256", "hashlib.sha256(image_path.encode()).hexdigest", "image_path.encode", "logger.warning", "hashlib.sha256(str(image_path).encode()).hexdigest", "str(image_path).encode", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hashlib.sha256(content).hexdigest()[:16]", "hashlib.sha256(image_path.encode()).hexdigest()[:16]", "hashlib.sha256(str(image_path).encode()).hexdigest()[:16]" ], "awaits": false, "yields": false }, { "name": "compute_images_hash", "qualname": "compute_images_hash", "full_name": "vllm_mlx.mllm_cache.compute_images_hash", "kind": "function", "signature": "def compute_images_hash(images: list[str]) -> str", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "List of image paths/URLs" } ], "return_annotation": "str", "docstring": "Compute combined hash for multiple images.\n\nArgs:\n images: List of image paths/URLs\n\nReturns:\n Combined hash string", "summary": "Compute combined hash for multiple images.", "implementation": "Function `compute_images_hash` calls `compute_image_hash`, `'_'.join`, `sorted`, `hashlib.sha256(combined.encode()).hexdigest`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 164, "end_line": 179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L164-L179", "decorators": [], "calls": [ "compute_image_hash", "'_'.join", "sorted", "hashlib.sha256(combined.encode()).hexdigest", "hashlib.sha256", "combined.encode" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'no_images'", "hashlib.sha256(combined.encode()).hexdigest()[:16]" ], "awaits": false, "yields": false }, { "name": "MLLMPrefixCacheManager", "qualname": "MLLMPrefixCacheManager", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager", "kind": "class", "signature": "class MLLMPrefixCacheManager", "parameters": [ { "name": "max_entries", "kind": "positional or keyword", "annotation": "int", "default": "50", "required": false, "description": "Maximum number of cache entries (default: 50)" }, { "name": "max_memory_mb", "kind": "positional or keyword", "annotation": "int", "default": "2048", "required": false, "description": "Maximum memory in MB (default: 2048)" } ], "return_annotation": "MLLMPrefixCacheManager", "docstring": "LRU Cache manager for MLLM prefix states with vision embedding caching.\n\nImplements the LMCache approach for multimodal caching:\n1. Hash-based identification of image+prompt combinations\n2. Vision embedding caching (skip encoder on hit - saves 1-2s!)\n3. KV cache reuse for matching prefixes\n4. Token ID tracking for partial prefix matching\n5. Memory-based eviction (configurable limit)\n\nExample:\n >>> cache = MLLMPrefixCacheManager(max_memory_mb=2048)\n >>> # First request - cache miss, full computation\n >>> entry, match_len = cache.fetch([\"image.jpg\"], prompt, token_ids)\n >>> # ... run full forward pass ...\n >>> cache.store([\"image.jpg\"], prompt, vision_emb, kv_cache, token_ids)\n >>>\n >>> # Second request with same image - cache hit!\n >>> entry, match_len = cache.fetch([\"image.jpg\"], prompt, token_ids)\n >>> # entry.vision_embeddings available - skip encoder!\n >>> # match_len > 0 - skip prefix computation!\n\nPerformance (Gemma 3 27B, 256 image tokens):\n - Vision encoder: ~1.5s -> 0s (skip on hit)\n - Prefix computation: ~0.5s/1k tokens -> 0s (skip on match)\n - Multi-turn speedup: 8-12x for subsequent turns", "summary": "LRU Cache manager for MLLM prefix states with vision embedding caching.", "implementation": "Class `MLLMPrefixCacheManager` declares 14 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 182, "end_line": 448, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L182-L448", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLLMPrefixCacheManager.__init__", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.__init__", "kind": "method", "signature": "def __init__(self, max_entries: int=50, max_memory_mb: int=2048)", "parameters": [ { "name": "max_entries", "kind": "positional or keyword", "annotation": "int", "default": "50", "required": false, "description": "Maximum number of cache entries (default: 50)" }, { "name": "max_memory_mb", "kind": "positional or keyword", "annotation": "int", "default": "2048", "required": false, "description": "Maximum memory in MB (default: 2048)" } ], "return_annotation": "not annotated", "docstring": "Initialize MLLM prefix cache manager.\n\nArgs:\n max_entries: Maximum number of cache entries (default: 50)\n max_memory_mb: Maximum memory in MB (default: 2048)", "summary": "Initialize MLLM prefix cache manager.", "implementation": "Method `MLLMPrefixCacheManager.__init__` updates `self.max_size`, `self.max_memory`, `self._cache`, `self._current_memory`; calls `OrderedDict`, `MLLMCacheStats`.", "documented": true, "public": false, "addressable": true, "line": 211, "end_line": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L211-L227", "decorators": [], "calls": [ "OrderedDict", "MLLMCacheStats" ], "state_reads": [], "state_writes": [ "self.max_size", "self.max_memory", "self._cache", "self._current_memory", "self.stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_make_cache_key", "qualname": "MLLMPrefixCacheManager._make_cache_key", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager._make_cache_key", "kind": "method", "signature": "def _make_cache_key(self, images: list[str], prompt: str) -> str", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Create cache key from images and prompt.", "summary": "Create cache key from images and prompt.", "implementation": "Method `MLLMPrefixCacheManager._make_cache_key` calls `compute_images_hash`, `hashlib.sha256(prompt.encode()).hexdigest`, `hashlib.sha256`, `prompt.encode`; returns `f'{image_hash}_{prompt_hash}'`.", "documented": true, "public": false, "addressable": true, "line": 229, "end_line": 233, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L229-L233", "decorators": [], "calls": [ "compute_images_hash", "hashlib.sha256(prompt.encode()).hexdigest", "hashlib.sha256", "prompt.encode" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'{image_hash}_{prompt_hash}'" ], "awaits": false, "yields": false }, { "name": "_make_image_only_key", "qualname": "MLLMPrefixCacheManager._make_image_only_key", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager._make_image_only_key", "kind": "method", "signature": "def _make_image_only_key(self, images: list[str]) -> str", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Create cache key for image-only lookup (vision embedding reuse).", "summary": "Create cache key for image-only lookup (vision embedding reuse).", "implementation": "Method `MLLMPrefixCacheManager._make_image_only_key` calls `compute_images_hash`; returns `compute_images_hash(images)`.", "documented": true, "public": false, "addressable": true, "line": 235, "end_line": 237, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L235-L237", "decorators": [], "calls": [ "compute_images_hash" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "compute_images_hash(images)" ], "awaits": false, "yields": false }, { "name": "_evict_by_memory", "qualname": "MLLMPrefixCacheManager._evict_by_memory", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager._evict_by_memory", "kind": "method", "signature": "def _evict_by_memory(self, required_size: int) -> None", "parameters": [ { "name": "required_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Evict entries until we have enough memory.", "summary": "Evict entries until we have enough memory.", "implementation": "Method `MLLMPrefixCacheManager._evict_by_memory` updates `self._current_memory`, `self.stats.evictions`; calls `next`, `iter`, `self._cache.pop`, `logger.debug`.", "documented": true, "public": false, "addressable": true, "line": 239, "end_line": 246, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L239-L246", "decorators": [], "calls": [ "next", "iter", "self._cache.pop", "logger.debug" ], "state_reads": [ "self._current_memory", "self.max_memory", "self._cache", "self._cache.pop", "self.stats" ], "state_writes": [ "self._current_memory", "self.stats.evictions" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_evict_by_count", "qualname": "MLLMPrefixCacheManager._evict_by_count", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager._evict_by_count", "kind": "method", "signature": "def _evict_by_count(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Evict entries until we're under max_size.", "summary": "Evict entries until we're under max_size.", "implementation": "Method `MLLMPrefixCacheManager._evict_by_count` updates `self._current_memory`, `self.stats.evictions`; calls `len`, `next`, `iter`, `self._cache.pop`.", "documented": true, "public": false, "addressable": true, "line": 248, "end_line": 255, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L248-L255", "decorators": [], "calls": [ "len", "next", "iter", "self._cache.pop", "logger.debug" ], "state_reads": [ "self._cache", "self.max_size", "self._cache.pop", "self.stats" ], "state_writes": [ "self._current_memory", "self.stats.evictions" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "fetch", "qualname": "MLLMPrefixCacheManager.fetch", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.fetch", "kind": "method", "signature": "def fetch(self, images: list[str], prompt: str, token_ids: list[int] | None=None) -> tuple[MLLMPrefixCacheEntry | None, int]", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "List of image paths" }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt" }, { "name": "token_ids", "kind": "positional or keyword", "annotation": "list[int] | None", "default": "None", "required": false, "description": "Optional token IDs for prefix matching" } ], "return_annotation": "tuple[MLLMPrefixCacheEntry | None, int]", "docstring": "Fetch cached prefix state with prefix matching.\n\nThis is the main entry point for cache lookups. Returns both\nthe cache entry (if found) and the prefix match length.\n\nArgs:\n images: List of image paths\n prompt: Text prompt\n token_ids: Optional token IDs for prefix matching\n\nReturns:\n Tuple of (entry, prefix_match_length) where:\n - entry: The cache entry if found, None otherwise\n - prefix_match_length: Number of tokens that match (0 if miss)", "summary": "Fetch cached prefix state with prefix matching.", "implementation": "Method `MLLMPrefixCacheManager.fetch` updates `self.stats.total_queries`, `self.stats.hits`, `self.stats.image_cache_hits`, `self.stats.vision_encoder_skips`; calls `self._make_cache_key`, `self._cache.pop`, `entry.get_prefix_match_length`, `logger.debug`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 257, "end_line": 329, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L257-L329", "decorators": [], "calls": [ "self._make_cache_key", "self._cache.pop", "entry.get_prefix_match_length", "logger.debug", "self._make_image_only_key", "self._cache.items" ], "state_reads": [ "self.stats", "self._make_cache_key", "self._cache", "self._cache.pop", "self._make_image_only_key", "self._cache.items" ], "state_writes": [ "self.stats.total_queries", "self.stats.hits", "self.stats.image_cache_hits", "self.stats.vision_encoder_skips", "self.stats.partial_hits", "self.stats.tokens_saved", "self.stats.misses" ], "raises": [], "return_expressions": [ "(entry, match_length)", "(entry, 0)", "(None, 0)" ], "awaits": false, "yields": false }, { "name": "fetch_cache", "qualname": "MLLMPrefixCacheManager.fetch_cache", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.fetch_cache", "kind": "method", "signature": "def fetch_cache(self, images: list[str], prompt: str) -> tuple[list[Any] | None, bool]", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[list[Any] | None, bool]", "docstring": "Legacy API: Fetch cached KV state for image+prompt combination.\n\nFor backwards compatibility with existing code.", "summary": "Legacy API: Fetch cached KV state for image+prompt combination.", "implementation": "Method `MLLMPrefixCacheManager.fetch_cache` calls `self.fetch`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 331, "end_line": 345, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L331-L345", "decorators": [], "calls": [ "self.fetch" ], "state_reads": [ "self.fetch" ], "state_writes": [], "raises": [], "return_expressions": [ "(entry.kv_cache, True)", "(None, False)" ], "awaits": false, "yields": false }, { "name": "store", "qualname": "MLLMPrefixCacheManager.store", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.store", "kind": "method", "signature": "def store(self, images: list[str], prompt: str, vision_embeddings: Any, kv_cache: list[Any], token_ids: list[int], num_image_tokens: int=0, model_name: str='') -> None", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "List of image paths" }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt" }, { "name": "vision_embeddings", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Output of vision encoder (can be None for text-only)" }, { "name": "kv_cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Language model KV cache states" }, { "name": "token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Full token sequence" }, { "name": "num_image_tokens", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Number of image tokens (e.g., 256 for Gemma 3)" }, { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "''", "required": false, "description": "Model name for validation" } ], "return_annotation": "None", "docstring": "Store prefix state in cache.\n\nArgs:\n images: List of image paths\n prompt: Text prompt\n vision_embeddings: Output of vision encoder (can be None for text-only)\n kv_cache: Language model KV cache states\n token_ids: Full token sequence\n num_image_tokens: Number of image tokens (e.g., 256 for Gemma 3)\n model_name: Model name for validation", "summary": "Store prefix state in cache.", "implementation": "Method `MLLMPrefixCacheManager.store` updates `self._current_memory`; calls `self._make_cache_key`, `MLLMPrefixCacheEntry`, `compute_images_hash`, `hashlib.sha256(prompt.encode()).hexdigest`.", "documented": true, "public": true, "addressable": true, "line": 347, "end_line": 396, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L347-L396", "decorators": [], "calls": [ "self._make_cache_key", "MLLMPrefixCacheEntry", "compute_images_hash", "hashlib.sha256(prompt.encode()).hexdigest", "hashlib.sha256", "prompt.encode", "len", "self._evict_by_memory", "self._evict_by_count", "logger.debug" ], "state_reads": [ "self._make_cache_key", "self._evict_by_memory", "self._evict_by_count", "self._cache" ], "state_writes": [ "self._current_memory" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "store_cache", "qualname": "MLLMPrefixCacheManager.store_cache", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.store_cache", "kind": "method", "signature": "def store_cache(self, images: list[str], prompt: str, cache: list[Any] | None, num_tokens: int=0) -> None", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_tokens", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "None", "docstring": "Legacy API: Store KV cache for future reuse.\n\nFor backwards compatibility with existing code.", "summary": "Legacy API: Store KV cache for future reuse.", "implementation": "Method `MLLMPrefixCacheManager.store_cache` calls `isinstance`, `len`, `self.store`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 398, "end_line": 421, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L398-L421", "decorators": [], "calls": [ "isinstance", "len", "self.store" ], "state_reads": [ "self.store" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "MLLMPrefixCacheManager.get_stats", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict[str, Any]", "parameters": [], "return_annotation": "dict[str, Any]", "docstring": "Get cache statistics.", "summary": "Get cache statistics.", "implementation": "Method `MLLMPrefixCacheManager.get_stats` calls `self.stats.to_dict`, `len`; returns `stats`.", "documented": true, "public": true, "addressable": true, "line": 423, "end_line": 430, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L423-L430", "decorators": [], "calls": [ "self.stats.to_dict", "len" ], "state_reads": [ "self.stats.to_dict", "self.stats", "self._cache", "self.max_size", "self._current_memory", "self.max_memory" ], "state_writes": [], "raises": [], "return_expressions": [ "stats" ], "awaits": false, "yields": false }, { "name": "reset_stats", "qualname": "MLLMPrefixCacheManager.reset_stats", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.reset_stats", "kind": "method", "signature": "def reset_stats(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset statistics counters.", "summary": "Reset statistics counters.", "implementation": "Method `MLLMPrefixCacheManager.reset_stats` updates `self.stats`; calls `MLLMCacheStats`.", "documented": true, "public": true, "addressable": true, "line": 432, "end_line": 434, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L432-L434", "decorators": [], "calls": [ "MLLMCacheStats" ], "state_reads": [], "state_writes": [ "self.stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "clear", "qualname": "MLLMPrefixCacheManager.clear", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear all cached entries and reset stats.", "summary": "Clear all cached entries and reset stats.", "implementation": "Method `MLLMPrefixCacheManager.clear` updates `self._current_memory`; calls `self._cache.clear`, `self.reset_stats`.", "documented": true, "public": true, "addressable": true, "line": 436, "end_line": 440, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L436-L440", "decorators": [], "calls": [ "self._cache.clear", "self.reset_stats" ], "state_reads": [ "self._cache.clear", "self._cache", "self.reset_stats" ], "state_writes": [ "self._current_memory" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "MLLMPrefixCacheManager.__len__", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.__len__", "kind": "method", "signature": "def __len__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return number of cached entries.", "summary": "Return number of cached entries.", "implementation": "Method `MLLMPrefixCacheManager.__len__` calls `len`; returns `len(self._cache)`.", "documented": true, "public": false, "addressable": true, "line": 442, "end_line": 444, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L442-L444", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self._cache)" ], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "MLLMPrefixCacheManager.__repr__", "full_name": "vllm_mlx.mllm_cache.MLLMPrefixCacheManager.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `MLLMPrefixCacheManager.__repr__` calls `len`; returns `f''`.", "implementation": "Method `MLLMPrefixCacheManager.__repr__` calls `len`; returns `f''`.", "documented": false, "public": false, "addressable": true, "line": 446, "end_line": 448, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_cache.py#L446-L448", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._current_memory" ], "state_writes": [], "raises": [], "return_expressions": [ "f''" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.mllm_scheduler", "path": "vllm_mlx/mllm_scheduler.py", "page_path": "reference/api/vllm_mlx/mllm_scheduler.md", "docstring": "MLLM Scheduler for multimodal continuous batching.\n\nThis scheduler handles Multimodal Language Model requests with continuous\nbatching support, following the same architecture as the LLM scheduler.\n\nKey features:\n- Batch processing of multiple MLLM requests\n- Vision embedding caching for repeated images\n- Step-based generation loop (like LLM scheduler)\n- Support for both streaming and non-streaming generation\n\nArchitecture:\n1. Requests arrive via add_request() -> waiting queue\n2. Scheduler moves requests from waiting to running (via MLLMBatchGenerator)\n3. step() method generates one token for ALL running requests\n4. Finished requests are removed and outputs returned", "summary": "MLLM Scheduler for multimodal continuous batching.", "line_count": 1242, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L1-L1242", "members": [ "logger", "MLLMSchedulerConfig", "MLLMRequest", "MLLMSchedulerOutput", "MLLMScheduler" ], "symbols": [ { "name": "MLLMSchedulerConfig", "qualname": "MLLMSchedulerConfig", "full_name": "vllm_mlx.mllm_scheduler.MLLMSchedulerConfig", "kind": "class", "signature": "class MLLMSchedulerConfig", "parameters": [ { "name": "max_num_seqs", "kind": "field", "annotation": "int", "default": "16", "required": false, "description": "Optional constructor field; defaults to `16`." }, { "name": "prefill_batch_size", "kind": "field", "annotation": "int", "default": "16", "required": false, "description": "Optional constructor field; defaults to `16`." }, { "name": "completion_batch_size", "kind": "field", "annotation": "int", "default": "16", "required": false, "description": "Optional constructor field; defaults to `16`." }, { "name": "prefill_step_size", "kind": "field", "annotation": "int", "default": "1024", "required": false, "description": "Optional constructor field; defaults to `1024`." }, { "name": "enable_vision_cache", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "vision_cache_size", "kind": "field", "annotation": "int", "default": "100", "required": false, "description": "Optional constructor field; defaults to `100`." }, { "name": "default_max_tokens", "kind": "field", "annotation": "int", "default": "256", "required": false, "description": "Optional constructor field; defaults to `256`." }, { "name": "default_video_fps", "kind": "field", "annotation": "float", "default": "2.0", "required": false, "description": "Optional constructor field; defaults to `2.0`." }, { "name": "cache_memory_mb", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_video_frames", "kind": "field", "annotation": "int", "default": "128", "required": false, "description": "Optional constructor field; defaults to `128`." }, { "name": "enable_mtp", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "mtp_num_draft_tokens", "kind": "field", "annotation": "int", "default": "1", "required": false, "description": "Optional constructor field; defaults to `1`." }, { "name": "enable_prefix_cache", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "use_memory_aware_cache", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "prefix_cache_memory_mb", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "kv_cache_quantization", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "kv_cache_quantization_bits", "kind": "field", "annotation": "int", "default": "8", "required": false, "description": "Optional constructor field; defaults to `8`." }, { "name": "kv_cache_quantization_group_size", "kind": "field", "annotation": "int", "default": "64", "required": false, "description": "Optional constructor field; defaults to `64`." }, { "name": "chunked_prefill_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "max_kv_size", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "ssd_cache_dir", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "ssd_cache_max_gb", "kind": "field", "annotation": "float", "default": "10.0", "required": false, "description": "Optional constructor field; defaults to `10.0`." } ], "return_annotation": "MLLMSchedulerConfig", "docstring": "Configuration for MLLM scheduler.", "summary": "Configuration for MLLM scheduler.", "implementation": "Class `MLLMSchedulerConfig` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 46, "end_line": 92, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L46-L92", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLLMRequest", "qualname": "MLLMRequest", "full_name": "vllm_mlx.mllm_scheduler.MLLMRequest", "kind": "class", "signature": "class MLLMRequest", "parameters": [ { "name": "request_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "images", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "videos", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "audio", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "sampling_params", "kind": "field", "annotation": "SamplingParams", "default": "field(default_factory=SamplingParams)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=SamplingParams)`." }, { "name": "arrival_time", "kind": "field", "annotation": "float", "default": "field(default_factory=time.time)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=time.time)`." }, { "name": "batch_uid", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "status", "kind": "field", "annotation": "RequestStatus", "default": "RequestStatus.WAITING", "required": false, "description": "Optional constructor field; defaults to `RequestStatus.WAITING`." }, { "name": "output_text", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "output_tokens", "kind": "field", "annotation": "List[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "finish_reason", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "num_prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "num_output_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_drafts", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_accepted", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "first_token_time", "kind": "field", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "MLLMRequest", "docstring": "Extended request for MLLM processing.\n\nIncludes all multimodal data needed for generation.", "summary": "Extended request for MLLM processing.", "implementation": "Class `MLLMRequest` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 96, "end_line": 127, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L96-L127", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLLMSchedulerOutput", "qualname": "MLLMSchedulerOutput", "full_name": "vllm_mlx.mllm_scheduler.MLLMSchedulerOutput", "kind": "class", "signature": "class MLLMSchedulerOutput", "parameters": [ { "name": "scheduled_request_ids", "kind": "field", "annotation": "List[str]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "num_scheduled_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "finished_request_ids", "kind": "field", "annotation": "Set[str]", "default": "field(default_factory=set)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=set)`." }, { "name": "outputs", "kind": "field", "annotation": "List[RequestOutput]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "has_work", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "MLLMSchedulerOutput", "docstring": "Output from a scheduling step.\n\nContains information about what was scheduled and results.", "summary": "Output from a scheduling step.", "implementation": "Class `MLLMSchedulerOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 131, "end_line": 147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L131-L147", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLLMScheduler", "qualname": "MLLMScheduler", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler", "kind": "class", "signature": "class MLLMScheduler", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM model" }, { "name": "processor", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM processor" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[MLLMSchedulerConfig]", "default": "None", "required": false, "description": "Scheduler configuration" } ], "return_annotation": "MLLMScheduler", "docstring": "Scheduler for Vision Language Model requests with continuous batching.\n\nThis scheduler manages the lifecycle of MLLM requests using the\nMLLMBatchGenerator for efficient batch processing:\n\n1. Requests arrive and are added to the waiting queue\n2. Scheduler moves requests from waiting to running (via batch generator)\n3. step() generates one token for ALL running requests simultaneously\n4. Finished requests are removed and outputs returned\n\nExample:\n >>> scheduler = MLLMScheduler(model, processor, config)\n >>> # Add requests\n >>> request_id = scheduler.add_request(\n ... prompt=\"What's in this image?\",\n ... images=[\"photo.jpg\"]\n ... )\n >>> # Run generation loop\n >>> while scheduler.has_requests():\n ... output = scheduler.step()\n ... for req_output in output.outputs:\n ... if req_output.finished:\n ... print(f\"Finished: {req_output.output_text}\")\n\nFor async usage with streaming:\n >>> await scheduler.start()\n >>> request_id = await scheduler.add_request_async(...)\n >>> async for output in scheduler.stream_outputs(request_id):\n ... print(output.new_text, end=\"\")", "summary": "Scheduler for Vision Language Model requests with continuous batching.", "implementation": "Class `MLLMScheduler` declares 24 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 150, "end_line": 1242, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L150-L1242", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLLMScheduler.__init__", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.__init__", "kind": "method", "signature": "def __init__(self, model: Any, processor: Any, config: Optional[MLLMSchedulerConfig]=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM model" }, { "name": "processor", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM processor" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[MLLMSchedulerConfig]", "default": "None", "required": false, "description": "Scheduler configuration" } ], "return_annotation": "not annotated", "docstring": "Initialize MLLM scheduler.\n\nArgs:\n model: The VLM model\n processor: The VLM processor\n config: Scheduler configuration", "summary": "Initialize MLLM scheduler.", "implementation": "Method `MLLMScheduler.__init__` updates `self.model`, `self.processor`, `self.config`, `self.model_config`; calls `MLLMSchedulerConfig`, `getattr`, `MultimodalProcessor`, `self._get_stop_tokens`.", "documented": true, "public": false, "addressable": true, "line": 183, "end_line": 248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L183-L248", "decorators": [], "calls": [ "MLLMSchedulerConfig", "getattr", "MultimodalProcessor", "self._get_stop_tokens", "deque", "set" ], "state_reads": [ "self.model_config", "self._get_stop_tokens" ], "state_writes": [ "self.model", "self.processor", "self.config", "self.model_config", "self.mm_processor", "self.stop_tokens", "self.batch_generator", "self.waiting", "self.running", "self.requests", "self.finished_req_ids", "self.request_id_to_uid", "self.uid_to_request_id", "self._detokenizer_pool", "self.output_queues", "self._running", "self._processing_task", "self._step_count", "self._clear_cache_interval", "self.num_requests_processed", "self.total_prompt_tokens", "self.total_completion_tokens" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_get_stop_tokens", "qualname": "MLLMScheduler._get_stop_tokens", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler._get_stop_tokens", "kind": "method", "signature": "def _get_stop_tokens(self) -> Set[int]", "parameters": [], "return_annotation": "Set[int]", "docstring": "Get stop token IDs from tokenizer and generation_config.json.", "summary": "Get stop token IDs from tokenizer and generation_config.json.", "implementation": "Method `MLLMScheduler._get_stop_tokens` calls `set`, `hasattr`, `isinstance`, `stop_tokens.update`; returns `stop_tokens`.", "documented": true, "public": false, "addressable": true, "line": 250, "end_line": 290, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L250-L290", "decorators": [], "calls": [ "set", "hasattr", "isinstance", "stop_tokens.update", "stop_tokens.add", "getattr", "Path", "gc_path.exists", "json.loads", "gc_path.read_text", "gc.get" ], "state_reads": [ "self.processor", "self.processor.tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "stop_tokens" ], "awaits": false, "yields": false }, { "name": "_ensure_batch_generator", "qualname": "MLLMScheduler._ensure_batch_generator", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler._ensure_batch_generator", "kind": "method", "signature": "def _ensure_batch_generator(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Ensure batch generator exists.", "summary": "Ensure batch generator exists.", "implementation": "Method `MLLMScheduler._ensure_batch_generator` updates `self.batch_generator`, `self._ssd_tier`; calls `make_sampler`, `MemoryCacheConfig`, `MLLMBatchGenerator`, `getattr`.", "documented": true, "public": false, "addressable": true, "line": 292, "end_line": 374, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L292-L374", "decorators": [], "calls": [ "make_sampler", "MemoryCacheConfig", "MLLMBatchGenerator", "getattr", "SSDCacheConfig", "SSDCacheTier", "self._ssd_tier.start_writer", "self._ssd_tier.reconcile", "prefix_cache.set_ssd_tier", "logger.info", "install_chunked_prefill_mllm", "hasattr", "install_mtp_mllm" ], "state_reads": [ "self.batch_generator", "self.config.enable_prefix_cache", "self.config", "self.config.use_memory_aware_cache", "self.config.prefix_cache_memory_mb", "self.config.kv_cache_quantization", "self.config.kv_cache_quantization_bits", "self.config.kv_cache_quantization_group_size", "self.model", "self.processor", "self.mm_processor", "self.config.default_max_tokens", "self.stop_tokens", "self.config.prefill_batch_size", "self.config.completion_batch_size", "self.config.prefill_step_size", "self.config.max_kv_size", "self.config.ssd_cache_dir", "self.config.ssd_cache_max_gb", "self._ssd_tier.start_writer", "self._ssd_tier", "self._ssd_tier.reconcile", "self.config.chunked_prefill_tokens", "self.config.enable_mtp", "self.batch_generator.language_model", "self.config.mtp_num_draft_tokens" ], "state_writes": [ "self.batch_generator", "self._ssd_tier" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "add_request", "qualname": "MLLMScheduler.add_request", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.add_request", "kind": "method", "signature": "def add_request(self, prompt: str, images: Optional[List[str]]=None, videos: Optional[List[str]]=None, audio: Optional[List[str]]=None, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, request_id: Optional[str]=None, **kwargs) -> str", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt (should be formatted with chat template)" }, { "name": "images", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of image inputs (paths, URLs, base64)" }, { "name": "videos", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of video inputs" }, { "name": "audio", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of audio inputs" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "request_id", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional custom request ID" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional generation parameters. ``logits_processors`` — list of callables ``(tokens, logits) -> logits`` applied during sampling (e.g. constrained JSON decoding)." } ], "return_annotation": "str", "docstring": "Add a multimodal request to the scheduler (sync version).\n\nArgs:\n prompt: Text prompt (should be formatted with chat template)\n images: List of image inputs (paths, URLs, base64)\n videos: List of video inputs\n audio: List of audio inputs\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n request_id: Optional custom request ID\n **kwargs: Additional generation parameters. ``logits_processors``\n — list of callables ``(tokens, logits) -> logits`` applied\n during sampling (e.g. constrained JSON decoding).\n\nReturns:\n Request ID for tracking", "summary": "Add a multimodal request to the scheduler (sync version).", "implementation": "Method `MLLMScheduler.add_request` calls `str`, `uuid.uuid4`, `SamplingParams`, `kwargs.pop`; returns `request_id`.", "documented": true, "public": true, "addressable": true, "line": 378, "end_line": 453, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L378-L453", "decorators": [], "calls": [ "str", "uuid.uuid4", "SamplingParams", "kwargs.pop", "MLLMRequest", "hasattr", "len", "tokenizer.encode", "self.waiting.append", "logger.debug" ], "state_reads": [ "self.processor", "self.processor.tokenizer", "self.requests", "self.waiting.append", "self.waiting" ], "state_writes": [], "raises": [], "return_expressions": [ "request_id" ], "awaits": false, "yields": false }, { "name": "abort_request", "qualname": "MLLMScheduler.abort_request", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.abort_request", "kind": "method", "signature": "def abort_request(self, request_id: str) -> bool", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The request ID to abort" } ], "return_annotation": "bool", "docstring": "Abort a request.\n\nArgs:\n request_id: The request ID to abort\n\nReturns:\n True if request was found and aborted", "summary": "Abort a request.", "implementation": "Method `MLLMScheduler.abort_request` updates `self.total_completion_tokens`, `self.total_prompt_tokens`; calls `self.requests.get`, `self.batch_generator.abort_prefill`, `self.waiting.remove`, `self.batch_generator.schedule_removal`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 455, "end_line": 532, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L455-L532", "decorators": [], "calls": [ "self.requests.get", "self.batch_generator.abort_prefill", "self.waiting.remove", "self.batch_generator.schedule_removal", "self.finished_req_ids.add", "self.requests.pop", "self._detokenizer_pool.pop", "self.output_queues[request_id].put_nowait", "logger.debug" ], "state_reads": [ "self.requests.get", "self.requests", "self.batch_generator", "self.batch_generator.abort_prefill", "self.waiting.remove", "self.waiting", "self.request_id_to_uid", "self.batch_generator.schedule_removal", "self.uid_to_request_id", "self.running", "self.finished_req_ids.add", "self.finished_req_ids", "self.requests.pop", "self._detokenizer_pool.pop", "self._detokenizer_pool", "self.output_queues" ], "state_writes": [ "self.total_completion_tokens", "self.total_prompt_tokens" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "has_requests", "qualname": "MLLMScheduler.has_requests", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.has_requests", "kind": "method", "signature": "def has_requests(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if there are any pending or running requests.", "summary": "Check if there are any pending or running requests.", "implementation": "Method `MLLMScheduler.has_requests` calls `bool`; returns `bool(self.waiting or self.running)`.", "documented": true, "public": true, "addressable": true, "line": 534, "end_line": 536, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L534-L536", "decorators": [], "calls": [ "bool" ], "state_reads": [ "self.waiting", "self.running" ], "state_writes": [], "raises": [], "return_expressions": [ "bool(self.waiting or self.running)" ], "awaits": false, "yields": false }, { "name": "get_num_waiting", "qualname": "MLLMScheduler.get_num_waiting", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.get_num_waiting", "kind": "method", "signature": "def get_num_waiting(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get number of waiting requests.", "summary": "Get number of waiting requests.", "implementation": "Method `MLLMScheduler.get_num_waiting` calls `len`; returns `len(self.waiting)`.", "documented": true, "public": true, "addressable": true, "line": 538, "end_line": 540, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L538-L540", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.waiting" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.waiting)" ], "awaits": false, "yields": false }, { "name": "get_num_running", "qualname": "MLLMScheduler.get_num_running", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.get_num_running", "kind": "method", "signature": "def get_num_running(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get number of running requests.", "summary": "Get number of running requests.", "implementation": "Method `MLLMScheduler.get_num_running` calls `len`; returns `len(self.running)`.", "documented": true, "public": true, "addressable": true, "line": 542, "end_line": 544, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L542-L544", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.running" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.running)" ], "awaits": false, "yields": false }, { "name": "_schedule_waiting", "qualname": "MLLMScheduler._schedule_waiting", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler._schedule_waiting", "kind": "method", "signature": "def _schedule_waiting(self) -> List[MLLMRequest]", "parameters": [], "return_annotation": "List[MLLMRequest]", "docstring": "Move requests from waiting queue to running.\n\nReturns:\n List of requests that were scheduled", "summary": "Move requests from waiting queue to running.", "implementation": "Method `MLLMScheduler._schedule_waiting` updates `self.total_prompt_tokens`; calls `self._ensure_batch_generator`, `len`, `self.waiting.popleft`, `MLLMBatchRequest`; returns `scheduled`.", "documented": true, "public": false, "addressable": true, "line": 546, "end_line": 597, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L546-L597", "decorators": [], "calls": [ "self._ensure_batch_generator", "len", "self.waiting.popleft", "MLLMBatchRequest", "batch_requests.append", "scheduled.append", "self.batch_generator.insert", "zip", "logger.debug" ], "state_reads": [ "self._ensure_batch_generator", "self.waiting", "self.running", "self.config.max_num_seqs", "self.config", "self.waiting.popleft", "self.batch_generator", "self.batch_generator.insert", "self.request_id_to_uid", "self.uid_to_request_id" ], "state_writes": [ "self.total_prompt_tokens" ], "raises": [], "return_expressions": [ "scheduled" ], "awaits": false, "yields": false }, { "name": "_process_batch_responses", "qualname": "MLLMScheduler._process_batch_responses", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler._process_batch_responses", "kind": "method", "signature": "def _process_batch_responses(self, responses: List[MLLMBatchResponse]) -> Tuple[List[RequestOutput], Set[str]]", "parameters": [ { "name": "responses", "kind": "positional or keyword", "annotation": "List[MLLMBatchResponse]", "default": "", "required": true, "description": "List of MLLMBatchResponse objects" } ], "return_annotation": "Tuple[List[RequestOutput], Set[str]]", "docstring": "Process responses from batch generator.\n\nArgs:\n responses: List of MLLMBatchResponse objects\n\nReturns:\n Tuple of (outputs, finished_request_ids)", "summary": "Process responses from batch generator.", "implementation": "Method `MLLMScheduler._process_batch_responses` updates `self.num_requests_processed`, `self.total_completion_tokens`; calls `set`, `hasattr`, `self.uid_to_request_id.get`, `self.running.get`; returns `(outputs, finished_ids)`.", "documented": true, "public": false, "addressable": true, "line": 599, "end_line": 716, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L599-L716", "decorators": [], "calls": [ "set", "hasattr", "self.uid_to_request_id.get", "self.running.get", "RequestOutput", "finished_ids.add", "logger.warning", "outputs.append", "request.output_tokens.append", "len", "time.time", "NaiveStreamingDetokenizer", "detok.add_token", "self._detokenizer_pool.pop", "detok.finalize", "tokenizer.decode", "logger.debug" ], "state_reads": [ "self.processor", "self.processor.tokenizer", "self.uid_to_request_id.get", "self.uid_to_request_id", "self.running.get", "self.running", "self._detokenizer_pool", "self._detokenizer_pool.pop" ], "state_writes": [ "self.num_requests_processed", "self.total_completion_tokens" ], "raises": [], "return_expressions": [ "(outputs, finished_ids)" ], "awaits": false, "yields": false }, { "name": "_cleanup_finished", "qualname": "MLLMScheduler._cleanup_finished", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler._cleanup_finished", "kind": "method", "signature": "def _cleanup_finished(self, finished_ids: Set[str]) -> None", "parameters": [ { "name": "finished_ids", "kind": "positional or keyword", "annotation": "Set[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Clean up finished requests.", "summary": "Clean up finished requests.", "implementation": "Method `MLLMScheduler._cleanup_finished` calls `self.requests.pop`, `self._detokenizer_pool.pop`, `self.finished_req_ids.add`, `mx.clear_cache`.", "documented": true, "public": false, "addressable": true, "line": 718, "end_line": 744, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L718-L744", "decorators": [], "calls": [ "self.requests.pop", "self._detokenizer_pool.pop", "self.finished_req_ids.add", "mx.clear_cache" ], "state_reads": [ "self.running", "self.requests.pop", "self.requests", "self.request_id_to_uid", "self.uid_to_request_id", "self._detokenizer_pool.pop", "self._detokenizer_pool", "self.finished_req_ids.add", "self.finished_req_ids" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "step", "qualname": "MLLMScheduler.step", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.step", "kind": "method", "signature": "def step(self) -> MLLMSchedulerOutput", "parameters": [], "return_annotation": "MLLMSchedulerOutput", "docstring": "Execute one scheduling step.\n\nThis method:\n1. Schedules waiting requests into the batch\n2. Runs one generation step via MLLMBatchGenerator\n3. Processes outputs and handles finished requests\n\nReturns:\n MLLMSchedulerOutput with results of this step", "summary": "Execute one scheduling step.", "implementation": "Method `MLLMScheduler.step` updates `self._step_count`, `self.finished_req_ids`; calls `MLLMSchedulerOutput`, `self.batch_generator.process_pending_removals`, `self._schedule_waiting`, `sum`; returns `output`.", "documented": true, "public": true, "addressable": true, "line": 746, "end_line": 813, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L746-L813", "decorators": [], "calls": [ "MLLMSchedulerOutput", "self.batch_generator.process_pending_removals", "self._schedule_waiting", "sum", "self.batch_generator.next", "self._process_batch_responses", "self.output_queues.get", "queue.put_nowait", "self._cleanup_finished", "mx.clear_cache", "len", "max", "set" ], "state_reads": [ "self.batch_generator", "self.batch_generator.process_pending_removals", "self._schedule_waiting", "self.running", "self.batch_generator.next", "self._process_batch_responses", "self.output_queues.get", "self.output_queues", "self._cleanup_finished", "self._clear_cache_interval", "self._step_count" ], "state_writes": [ "self._step_count", "self.finished_req_ids" ], "raises": [], "return_expressions": [ "output" ], "awaits": false, "yields": false }, { "name": "get_request", "qualname": "MLLMScheduler.get_request", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.get_request", "kind": "method", "signature": "def get_request(self, request_id: str) -> Optional[MLLMRequest]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[MLLMRequest]", "docstring": "Get a request by ID.", "summary": "Get a request by ID.", "implementation": "Method `MLLMScheduler.get_request` calls `self.requests.get`; returns `self.requests.get(request_id)`.", "documented": true, "public": true, "addressable": true, "line": 815, "end_line": 817, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L815-L817", "decorators": [], "calls": [ "self.requests.get" ], "state_reads": [ "self.requests.get", "self.requests" ], "state_writes": [], "raises": [], "return_expressions": [ "self.requests.get(request_id)" ], "awaits": false, "yields": false }, { "name": "remove_finished_request", "qualname": "MLLMScheduler.remove_finished_request", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.remove_finished_request", "kind": "method", "signature": "def remove_finished_request(self, request_id: str) -> Optional[MLLMRequest]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[MLLMRequest]", "docstring": "Remove a finished request from tracking.", "summary": "Remove a finished request from tracking.", "implementation": "Method `MLLMScheduler.remove_finished_request` calls `self.requests.pop`; returns `self.requests.pop(request_id, None)`.", "documented": true, "public": true, "addressable": true, "line": 819, "end_line": 821, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L819-L821", "decorators": [], "calls": [ "self.requests.pop" ], "state_reads": [ "self.requests.pop", "self.requests" ], "state_writes": [], "raises": [], "return_expressions": [ "self.requests.pop(request_id, None)" ], "awaits": false, "yields": false }, { "name": "start", "qualname": "MLLMScheduler.start", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.start", "kind": "method", "signature": "async def start(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the async scheduler processing loop.", "summary": "Start the async scheduler processing loop.", "implementation": "Method `MLLMScheduler.start` updates `self._running`, `self._processing_task`; calls `asyncio.create_task`, `self._process_loop`, `logger.info`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 825, "end_line": 834, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L825-L834", "decorators": [], "calls": [ "asyncio.create_task", "self._process_loop", "logger.info" ], "state_reads": [ "self._running", "self._process_loop", "self.config.max_num_seqs", "self.config" ], "state_writes": [ "self._running", "self._processing_task" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "stop", "qualname": "MLLMScheduler.stop", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.stop", "kind": "method", "signature": "async def stop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop the scheduler.", "summary": "Stop the scheduler.", "implementation": "Method `MLLMScheduler.stop` updates `self._running`, `self.batch_generator`; calls `self._processing_task.cancel`, `self.batch_generator.close`, `logger.info`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 836, "end_line": 850, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L836-L850", "decorators": [], "calls": [ "self._processing_task.cancel", "self.batch_generator.close", "logger.info" ], "state_reads": [ "self._processing_task", "self._processing_task.cancel", "self.batch_generator", "self.batch_generator.close" ], "state_writes": [ "self._running", "self.batch_generator" ], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_process_loop", "qualname": "MLLMScheduler._process_loop", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler._process_loop", "kind": "method", "signature": "async def _process_loop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Main async processing loop.\n\nMLLM models are loaded on the server/event-loop thread, so their MLX\narrays and cache state must be consumed on that same thread. Unlike\nthe text-only EngineCore path, moving MLLM prefill to a worker crosses\nMLX stream ownership and can fail with \"no Stream in current thread\".\n\nText-only preprocessing (Jinja2 template rendering + tokenization) is\nrun BEFORE ``step()`` with ``await asyncio.sleep(0)`` yields between\neach request. This prevents long preprocessing (10-30+ s for 40K+\ntoken conversations) from blocking health checks and new connections.", "summary": "Main async processing loop.", "implementation": "Method `MLLMScheduler._process_loop` calls `asyncio.get_running_loop`, `list`, `getattr`, `time.perf_counter`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 852, "end_line": 947, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L852-L947", "decorators": [], "calls": [ "asyncio.get_running_loop", "list", "getattr", "time.perf_counter", "loop.run_in_executor", "logger.info", "logger.error", "self.has_requests", "_ensure_streams_bound", "self.step", "logger.warning", "len", "range", "asyncio.sleep" ], "state_reads": [ "self._running", "self.batch_generator", "self.has_requests", "self.step", "self.waiting", "self.running" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_ensure_streams_bound", "qualname": "MLLMScheduler._process_loop._ensure_streams_bound", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler._process_loop._ensure_streams_bound", "kind": "nested function", "signature": "def _ensure_streams_bound() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `MLLMScheduler._process_loop._ensure_streams_bound` calls `bind_generation_streams`.", "implementation": "Nested Function `MLLMScheduler._process_loop._ensure_streams_bound` calls `bind_generation_streams`.", "documented": false, "public": false, "addressable": false, "line": 867, "end_line": 871, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L867-L871", "decorators": [], "calls": [ "bind_generation_streams" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "add_request_async", "qualname": "MLLMScheduler.add_request_async", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.add_request_async", "kind": "method", "signature": "async def add_request_async(self, prompt: str, images: Optional[List[str]]=None, videos: Optional[List[str]]=None, audio: Optional[List[str]]=None, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, **kwargs) -> str", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt" }, { "name": "images", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of image inputs" }, { "name": "videos", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of video inputs" }, { "name": "audio", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of audio inputs" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters" } ], "return_annotation": "str", "docstring": "Add a multimodal request (async version with output queue).\n\nArgs:\n prompt: Text prompt\n images: List of image inputs\n videos: List of video inputs\n audio: List of audio inputs\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling\n **kwargs: Additional parameters\n\nReturns:\n Request ID for tracking", "summary": "Add a multimodal request (async version with output queue).", "implementation": "Method `MLLMScheduler.add_request_async` calls `self.add_request`, `asyncio.Queue`; returns `request_id`.", "documented": true, "public": true, "addressable": true, "line": 949, "end_line": 990, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L949-L990", "decorators": [], "calls": [ "self.add_request", "asyncio.Queue" ], "state_reads": [ "self.add_request", "self.output_queues" ], "state_writes": [], "raises": [], "return_expressions": [ "request_id" ], "awaits": false, "yields": false }, { "name": "stream_outputs", "qualname": "MLLMScheduler.stream_outputs", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.stream_outputs", "kind": "method", "signature": "async def stream_outputs(self, request_id: str) -> AsyncIterator[RequestOutput]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The request ID to stream" } ], "return_annotation": "AsyncIterator[RequestOutput]", "docstring": "Stream outputs for a request.\n\nArgs:\n request_id: The request ID to stream\n\nYields:\n RequestOutput objects as tokens are generated", "summary": "Stream outputs for a request.", "implementation": "Method `MLLMScheduler.stream_outputs` calls `self.output_queues.get`, `output_queue.get`, `logger.info`, `self.abort_request`; awaits asynchronous work; yields values incrementally; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 992, "end_line": 1027, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L992-L1027", "decorators": [], "calls": [ "self.output_queues.get", "output_queue.get", "logger.info", "self.abort_request" ], "state_reads": [ "self.output_queues.get", "self.output_queues", "self.abort_request" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": true }, { "name": "generate", "qualname": "MLLMScheduler.generate", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.generate", "kind": "method", "signature": "async def generate(self, prompt: str, images: Optional[List[str]]=None, videos: Optional[List[str]]=None, audio: Optional[List[str]]=None, **kwargs) -> RequestOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt" }, { "name": "images", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Image inputs" }, { "name": "videos", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Video inputs" }, { "name": "audio", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Audio inputs" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Generation parameters" } ], "return_annotation": "RequestOutput", "docstring": "Generate complete output for a request (non-streaming).\n\nArgs:\n prompt: Text prompt\n images: Image inputs\n videos: Video inputs\n audio: Audio inputs\n **kwargs: Generation parameters\n\nReturns:\n Final RequestOutput", "summary": "Generate complete output for a request (non-streaming).", "implementation": "Method `MLLMScheduler.generate` calls `self.add_request_async`, `self.stream_outputs`, `RequestOutput`; awaits asynchronous work; returns `final_output`.", "documented": true, "public": true, "addressable": true, "line": 1029, "end_line": 1078, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L1029-L1078", "decorators": [], "calls": [ "self.add_request_async", "self.stream_outputs", "RequestOutput" ], "state_reads": [ "self.add_request_async", "self.stream_outputs", "self.requests" ], "state_writes": [], "raises": [], "return_expressions": [ "final_output" ], "awaits": true, "yields": false }, { "name": "get_running_requests_info", "qualname": "MLLMScheduler.get_running_requests_info", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.get_running_requests_info", "kind": "method", "signature": "def get_running_requests_info(self) -> List[Dict[str, Any]]", "parameters": [], "return_annotation": "List[Dict[str, Any]]", "docstring": "Per-request details for status endpoint.", "summary": "Per-request details for status endpoint.", "implementation": "Method `MLLMScheduler.get_running_requests_info` calls `time.time`, `result.append`, `round`, `self.running.values`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 1082, "end_line": 1151, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L1082-L1151", "decorators": [], "calls": [ "time.time", "result.append", "round", "self.running.values", "self.batch_generator.get_prefill_progress", "min" ], "state_reads": [ "self.waiting", "self.running.values", "self.running", "self.batch_generator", "self.batch_generator.get_prefill_progress" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "MLLMScheduler.get_stats", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.get_stats", "kind": "method", "signature": "def get_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get scheduler statistics.", "summary": "Get scheduler statistics.", "implementation": "Method `MLLMScheduler.get_stats` calls `len`, `self.get_running_requests_info`, `self.batch_generator.stats`, `batch_stats.to_dict`; returns `stats`.", "documented": true, "public": true, "addressable": true, "line": 1153, "end_line": 1204, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L1153-L1204", "decorators": [], "calls": [ "len", "self.get_running_requests_info", "self.batch_generator.stats", "batch_stats.to_dict", "self.batch_generator.get_vision_cache_stats", "hasattr", "self.batch_generator.get_mtp_stats", "mx.metal.is_available", "round", "mx.get_active_memory", "mx.get_peak_memory", "mx.get_cache_memory", "self.batch_generator.get_prefix_cache_stats" ], "state_reads": [ "self.waiting", "self.running", "self.finished_req_ids", "self.num_requests_processed", "self.total_prompt_tokens", "self.total_completion_tokens", "self.get_running_requests_info", "self.batch_generator", "self.batch_generator.stats", "self.batch_generator.get_vision_cache_stats", "self.batch_generator.get_mtp_stats", "self.batch_generator.get_prefix_cache_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "stats" ], "awaits": false, "yields": false }, { "name": "clear_runtime_caches", "qualname": "MLLMScheduler.clear_runtime_caches", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.clear_runtime_caches", "kind": "method", "signature": "def clear_runtime_caches(self) -> Dict[str, bool]", "parameters": [], "return_annotation": "Dict[str, bool]", "docstring": "Clear runtime caches without resetting scheduler/request state.", "summary": "Clear runtime caches without resetting scheduler/request state.", "implementation": "Method `MLLMScheduler.clear_runtime_caches` calls `self.vision_cache.clear`, `self.batch_generator.prefix_cache.clear`; returns `cleared`.", "documented": true, "public": true, "addressable": true, "line": 1206, "end_line": 1221, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L1206-L1221", "decorators": [], "calls": [ "self.vision_cache.clear", "self.batch_generator.prefix_cache.clear" ], "state_reads": [ "self.vision_cache", "self.vision_cache.clear", "self.batch_generator", "self.batch_generator.prefix_cache", "self.batch_generator.prefix_cache.clear" ], "state_writes": [], "raises": [], "return_expressions": [ "cleared" ], "awaits": false, "yields": false }, { "name": "reset", "qualname": "MLLMScheduler.reset", "full_name": "vllm_mlx.mllm_scheduler.MLLMScheduler.reset", "kind": "method", "signature": "def reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset the scheduler state.", "summary": "Reset the scheduler state.", "implementation": "Method `MLLMScheduler.reset` updates `self.batch_generator`; calls `list`, `self.requests.keys`, `self.abort_request`, `self.waiting.clear`.", "documented": true, "public": true, "addressable": true, "line": 1223, "end_line": 1242, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mllm_scheduler.py#L1223-L1242", "decorators": [], "calls": [ "list", "self.requests.keys", "self.abort_request", "self.waiting.clear", "self.running.clear", "self.requests.clear", "self.finished_req_ids.clear", "self.request_id_to_uid.clear", "self.uid_to_request_id.clear", "self._detokenizer_pool.clear", "self.batch_generator.close", "self.vision_cache.clear" ], "state_reads": [ "self.requests.keys", "self.requests", "self.abort_request", "self.waiting.clear", "self.waiting", "self.running.clear", "self.running", "self.requests.clear", "self.finished_req_ids.clear", "self.finished_req_ids", "self.request_id_to_uid.clear", "self.request_id_to_uid", "self.uid_to_request_id.clear", "self.uid_to_request_id", "self._detokenizer_pool.clear", "self._detokenizer_pool", "self.batch_generator", "self.batch_generator.close", "self.vision_cache", "self.vision_cache.clear" ], "state_writes": [ "self.batch_generator" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.mlx_streams", "path": "vllm_mlx/mlx_streams.py", "page_path": "reference/api/vllm_mlx/mlx_streams.md", "docstring": "Helpers for binding MLX generation streams to worker threads.", "summary": "Helpers for binding MLX generation streams to worker threads.", "line_count": 39, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mlx_streams.py#L1-L39", "members": [ "_STREAM_REBIND_LOCK", "bind_generation_streams" ], "symbols": [ { "name": "bind_generation_streams", "qualname": "bind_generation_streams", "full_name": "vllm_mlx.mlx_streams.bind_generation_streams", "kind": "function", "signature": "def bind_generation_streams(module_names: Iterable[str]=('mlx_lm.generate', 'mlx_vlm.generate')) -> object", "parameters": [ { "name": "module_names", "kind": "positional or keyword", "annotation": "Iterable[str]", "default": "('mlx_lm.generate', 'mlx_vlm.generate')", "required": false, "description": "Optional positional or keyword input; defaults to `('mlx_lm.generate', 'mlx_vlm.generate')`." } ], "return_annotation": "object", "docstring": "Bind mlx-lm/mlx-vlm generation streams to the current thread.\n\nMLX streams are thread-local. If a model is loaded on one thread and\ngeneration runs on another, module-level generation streams created during\nimport can point at a stream that does not exist in the worker thread.\n\nThis intentionally creates a fresh stream for the current worker call and\nreplaces module-level generation_stream handles under a process-local lock.\nIt is an admission/ownership fix, not a batching optimization; callers\nshould invoke it at worker-entry boundaries rather than inside token loops.", "summary": "Bind mlx-lm/mlx-vlm generation streams to the current thread.", "implementation": "Function `bind_generation_streams` calls `mx.new_stream`, `mx.default_device`, `mx.set_default_stream`, `importlib.import_module`; returns `default_stream`.", "documented": true, "public": true, "addressable": true, "line": 15, "end_line": 39, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/mlx_streams.py#L15-L39", "decorators": [], "calls": [ "mx.new_stream", "mx.default_device", "mx.set_default_stream", "importlib.import_module", "hasattr", "setattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "default_stream" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.model_registry", "path": "vllm_mlx/model_registry.py", "page_path": "reference/api/vllm_mlx/model_registry.md", "docstring": "Registry-backed multi-model serving with memory-budget eviction.\n\nThe registry maps OpenAI-compatible ``model`` names to concrete local paths or\ndeclared HuggingFace IDs. Models are loaded lazily, optionally preloaded, and\nevicted according to a memory-budget policy with configurable wait/fail/preempt\nbehaviour.", "summary": "Registry-backed multi-model serving with memory-budget eviction.", "line_count": 1201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1-L1201", "members": [ "logger", "ModelOwnershipError", "_ModelOwnershipRegistry", "_ownership_registry", "get_registry", "ContentionStrategy", "EngineFactory", "RegistryServeDefaults", "ContentionPolicy", "RegistryManagerConfig", "RegisteredModel", "ResolvedModelConfig", "LoadedModel", "PendingLoad", "ModelLease", "_clone_scheduler_config", "_parse_memory_budget_bytes", "_safe_available_memory_bytes", "_device_working_set_bytes", "MemoryBudgetReport", "build_memory_budget_report", "log_memory_budget_report", "_estimate_model_bytes_from_source", "load_registry_config", "ModelManager" ], "symbols": [ { "name": "ModelOwnershipError", "qualname": "ModelOwnershipError", "full_name": "vllm_mlx.model_registry.ModelOwnershipError", "kind": "class", "signature": "class ModelOwnershipError(RuntimeError)", "parameters": [], "return_annotation": "ModelOwnershipError", "docstring": "Raised when an EngineCore attempts to use a model already in use.", "summary": "Raised when an EngineCore attempts to use a model already in use.", "implementation": "Class `ModelOwnershipError` derives from `RuntimeError` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 38, "end_line": 39, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L38-L39", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_ModelOwnershipRegistry", "qualname": "_ModelOwnershipRegistry", "full_name": "vllm_mlx.model_registry._ModelOwnershipRegistry", "kind": "class", "signature": "class _ModelOwnershipRegistry", "parameters": [], "return_annotation": "_ModelOwnershipRegistry", "docstring": "Process-local model ownership guard used by EngineCore.", "summary": "Process-local model ownership guard used by EngineCore.", "implementation": "Class `_ModelOwnershipRegistry` declares 5 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 42, "end_line": 82, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L42-L82", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_ModelOwnershipRegistry.__init__", "full_name": "vllm_mlx.model_registry._ModelOwnershipRegistry.__init__", "kind": "method", "signature": "def __init__(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `_ModelOwnershipRegistry.__init__` updates `self._owners`.", "implementation": "Method `_ModelOwnershipRegistry.__init__` updates `self._owners`.", "documented": false, "public": false, "addressable": true, "line": 45, "end_line": 46, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L45-L46", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._owners" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "acquire", "qualname": "_ModelOwnershipRegistry.acquire", "full_name": "vllm_mlx.model_registry._ModelOwnershipRegistry.acquire", "kind": "method", "signature": "def acquire(self, *, model: Any, engine: Any, engine_id: str, force: bool=True) -> None", "parameters": [ { "name": "model", "kind": "keyword-only", "annotation": "Any", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "engine", "kind": "keyword-only", "annotation": "Any", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "engine_id", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "force", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "None", "docstring": "", "summary": "Method `_ModelOwnershipRegistry.acquire` calls `id`, `self._owners.get`, `ModelOwnershipError`; can raise `ModelOwnershipError`.", "implementation": "Method `_ModelOwnershipRegistry.acquire` calls `id`, `self._owners.get`, `ModelOwnershipError`; can raise `ModelOwnershipError`.", "documented": false, "public": false, "addressable": true, "line": 48, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L48-L63", "decorators": [], "calls": [ "id", "self._owners.get", "ModelOwnershipError" ], "state_reads": [ "self._owners.get", "self._owners" ], "state_writes": [], "raises": [ "ModelOwnershipError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "release", "qualname": "_ModelOwnershipRegistry.release", "full_name": "vllm_mlx.model_registry._ModelOwnershipRegistry.release", "kind": "method", "signature": "def release(self, model: Any, engine_id: str) -> None", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `_ModelOwnershipRegistry.release` calls `id`, `self._owners.get`, `self._owners.pop`.", "implementation": "Method `_ModelOwnershipRegistry.release` calls `id`, `self._owners.get`, `self._owners.pop`.", "documented": false, "public": false, "addressable": true, "line": 65, "end_line": 69, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L65-L69", "decorators": [], "calls": [ "id", "self._owners.get", "self._owners.pop" ], "state_reads": [ "self._owners.get", "self._owners", "self._owners.pop" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_owned", "qualname": "_ModelOwnershipRegistry.is_owned", "full_name": "vllm_mlx.model_registry._ModelOwnershipRegistry.is_owned", "kind": "method", "signature": "def is_owned(self, model: Any) -> tuple[bool, str | None]", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str | None]", "docstring": "", "summary": "Method `_ModelOwnershipRegistry.is_owned` calls `id`, `self._owners.get`; has 2 explicit return paths.", "implementation": "Method `_ModelOwnershipRegistry.is_owned` calls `id`, `self._owners.get`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 71, "end_line": 76, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L71-L76", "decorators": [], "calls": [ "id", "self._owners.get" ], "state_reads": [ "self._owners.get", "self._owners" ], "state_writes": [], "raises": [], "return_expressions": [ "(True, owner)", "(False, None)" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "_ModelOwnershipRegistry.get_stats", "full_name": "vllm_mlx.model_registry._ModelOwnershipRegistry.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict[str, Any]", "parameters": [], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Method `_ModelOwnershipRegistry.get_stats` calls `len`; returns `{'total_entries': len(self._owners), 'active_owners': len(self._owners)}`.", "implementation": "Method `_ModelOwnershipRegistry.get_stats` calls `len`; returns `{'total_entries': len(self._owners), 'active_owners': len(self._owners)}`.", "documented": false, "public": false, "addressable": true, "line": 78, "end_line": 82, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L78-L82", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._owners" ], "state_writes": [], "raises": [], "return_expressions": [ "{'total_entries': len(self._owners), 'active_owners': len(self._owners)}" ], "awaits": false, "yields": false }, { "name": "get_registry", "qualname": "get_registry", "full_name": "vllm_mlx.model_registry.get_registry", "kind": "function", "signature": "def get_registry() -> _ModelOwnershipRegistry", "parameters": [], "return_annotation": "_ModelOwnershipRegistry", "docstring": "Return the global model ownership registry used by EngineCore.", "summary": "Return the global model ownership registry used by EngineCore.", "implementation": "Function `get_registry` returns `_ownership_registry`.", "documented": true, "public": true, "addressable": true, "line": 88, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L88-L90", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_ownership_registry" ], "awaits": false, "yields": false }, { "name": "RegistryServeDefaults", "qualname": "RegistryServeDefaults", "full_name": "vllm_mlx.model_registry.RegistryServeDefaults", "kind": "class", "signature": "class RegistryServeDefaults", "parameters": [ { "name": "continuous_batching", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "force_mllm", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "enable_mtp", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prefill_step_size", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_enabled", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_threshold", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_keep_pct", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_backbone_pct", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_draft_model", "kind": "field", "annotation": "str | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "stream_interval", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "gpu_memory_utilization", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "scheduler_config", "kind": "field", "annotation": "SchedulerConfig | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "max_tokens", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "download_config", "kind": "field", "annotation": "DownloadConfig", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "RegistryServeDefaults", "docstring": "Global serve defaults inherited by registry entries.", "summary": "Global serve defaults inherited by registry entries.", "implementation": "Class `RegistryServeDefaults` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 109, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L109-L125", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ContentionPolicy", "qualname": "ContentionPolicy", "full_name": "vllm_mlx.model_registry.ContentionPolicy", "kind": "class", "signature": "class ContentionPolicy", "parameters": [ { "name": "strategy", "kind": "field", "annotation": "ContentionStrategy", "default": "'wait_then_fail'", "required": false, "description": "Optional constructor field; defaults to `'wait_then_fail'`." }, { "name": "wait_timeout_s", "kind": "field", "annotation": "float | None", "default": "30.0", "required": false, "description": "Optional constructor field; defaults to `30.0`." }, { "name": "preempt_after_s", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ContentionPolicy", "docstring": "Policy used when a new model cannot fit inside the memory budget.", "summary": "Policy used when a new model cannot fit inside the memory budget.", "implementation": "Class `ContentionPolicy` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 129, "end_line": 134, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L129-L134", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RegistryManagerConfig", "qualname": "RegistryManagerConfig", "full_name": "vllm_mlx.model_registry.RegistryManagerConfig", "kind": "class", "signature": "class RegistryManagerConfig", "parameters": [ { "name": "memory_budget_bytes", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "policy", "kind": "field", "annotation": "ContentionPolicy", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "RegistryManagerConfig", "docstring": "Global registry manager configuration.", "summary": "Global registry manager configuration.", "implementation": "Class `RegistryManagerConfig` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 138, "end_line": 142, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L138-L142", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RegisteredModel", "qualname": "RegisteredModel", "full_name": "vllm_mlx.model_registry.RegisteredModel", "kind": "class", "signature": "class RegisteredModel", "parameters": [ { "name": "name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "source", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "preload", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "continuous_batching", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "force_mllm", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "enable_mtp", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prefill_step_size", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_enabled", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_threshold", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_keep_pct", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_backbone_pct", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "specprefill_draft_model", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "stream_interval", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "gpu_memory_utilization", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "estimated_memory_bytes", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "RegisteredModel", "docstring": "One configured model entry.", "summary": "One configured model entry.", "implementation": "Class `RegisteredModel` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 146, "end_line": 163, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L146-L163", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ResolvedModelConfig", "qualname": "ResolvedModelConfig", "full_name": "vllm_mlx.model_registry.ResolvedModelConfig", "kind": "class", "signature": "class ResolvedModelConfig", "parameters": [ { "name": "entry", "kind": "field", "annotation": "RegisteredModel", "default": "", "required": true, "description": "Required constructor field." }, { "name": "resolved_source", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "continuous_batching", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "force_mllm", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "enable_mtp", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prefill_step_size", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_enabled", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_threshold", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_keep_pct", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_backbone_pct", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "specprefill_draft_model", "kind": "field", "annotation": "str | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "stream_interval", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "gpu_memory_utilization", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "scheduler_config", "kind": "field", "annotation": "SchedulerConfig | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "estimated_memory_bytes", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ResolvedModelConfig", "docstring": "Effective configuration for a loaded model.", "summary": "Effective configuration for a loaded model.", "implementation": "Class `ResolvedModelConfig` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 167, "end_line": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L167-L184", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "LoadedModel", "qualname": "LoadedModel", "full_name": "vllm_mlx.model_registry.LoadedModel", "kind": "class", "signature": "class LoadedModel", "parameters": [ { "name": "config", "kind": "field", "annotation": "ResolvedModelConfig", "default": "", "required": true, "description": "Required constructor field." }, { "name": "engine", "kind": "field", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required constructor field." }, { "name": "loaded_at", "kind": "field", "annotation": "float", "default": "field(default_factory=time.time)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=time.time)`." }, { "name": "last_used_at", "kind": "field", "annotation": "float", "default": "field(default_factory=time.time)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=time.time)`." }, { "name": "active_requests", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "active_tasks", "kind": "field", "annotation": "set[asyncio.Task[Any]]", "default": "field(default_factory=set)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=set)`." }, { "name": "preempting", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "LoadedModel", "docstring": "Runtime state for a loaded engine.", "summary": "Runtime state for a loaded engine.", "implementation": "Class `LoadedModel` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 188, "end_line": 197, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L188-L197", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "PendingLoad", "qualname": "PendingLoad", "full_name": "vllm_mlx.model_registry.PendingLoad", "kind": "class", "signature": "class PendingLoad", "parameters": [ { "name": "model_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "required_bytes", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "future", "kind": "field", "annotation": "asyncio.Future[LoadedModel]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "PendingLoad", "docstring": "A reserved model load in progress.", "summary": "A reserved model load in progress.", "implementation": "Class `PendingLoad` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 201, "end_line": 206, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L201-L206", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ModelLease", "qualname": "ModelLease", "full_name": "vllm_mlx.model_registry.ModelLease", "kind": "class", "signature": "class ModelLease", "parameters": [ { "name": "manager", "kind": "field", "annotation": "'ModelManager | None'", "default": "", "required": true, "description": "Required constructor field." }, { "name": "model_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "engine", "kind": "field", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required constructor field." }, { "name": "release_cb", "kind": "field", "annotation": "Callable[[], Awaitable[None]]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "ModelLease", "docstring": "Active lease for a loaded model.", "summary": "Active lease for a loaded model.", "implementation": "Class `ModelLease` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 210, "end_line": 231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L210-L231", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "release", "qualname": "ModelLease.release", "full_name": "vllm_mlx.model_registry.ModelLease.release", "kind": "method", "signature": "async def release(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Release this lease once and allow the model to become evictable.", "summary": "Release this lease once and allow the model to become evictable.", "implementation": "Method `ModelLease.release` updates `self.manager`; calls `self.release_cb`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 218, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L218-L225", "decorators": [], "calls": [ "self.release_cb" ], "state_reads": [ "self.manager", "self.release_cb" ], "state_writes": [ "self.manager" ], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "__aenter__", "qualname": "ModelLease.__aenter__", "full_name": "vllm_mlx.model_registry.ModelLease.__aenter__", "kind": "method", "signature": "async def __aenter__(self) -> 'ModelLease'", "parameters": [], "return_annotation": "'ModelLease'", "docstring": "", "summary": "Method `ModelLease.__aenter__` returns `self`.", "implementation": "Method `ModelLease.__aenter__` returns `self`.", "documented": false, "public": false, "addressable": true, "line": 227, "end_line": 228, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L227-L228", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "self" ], "awaits": false, "yields": false }, { "name": "__aexit__", "qualname": "ModelLease.__aexit__", "full_name": "vllm_mlx.model_registry.ModelLease.__aexit__", "kind": "method", "signature": "async def __aexit__(self, exc_type, exc, tb) -> None", "parameters": [ { "name": "exc_type", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "exc", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tb", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ModelLease.__aexit__` calls `self.release`; awaits asynchronous work.", "implementation": "Method `ModelLease.__aexit__` calls `self.release`; awaits asynchronous work.", "documented": false, "public": false, "addressable": true, "line": 230, "end_line": 231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L230-L231", "decorators": [], "calls": [ "self.release" ], "state_reads": [ "self.release" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_clone_scheduler_config", "qualname": "_clone_scheduler_config", "full_name": "vllm_mlx.model_registry._clone_scheduler_config", "kind": "function", "signature": "def _clone_scheduler_config(config: SchedulerConfig | None) -> SchedulerConfig | None", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "SchedulerConfig | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "SchedulerConfig | None", "docstring": "Clone a SchedulerConfig so per-model overrides do not mutate globals.", "summary": "Clone a SchedulerConfig so per-model overrides do not mutate globals.", "implementation": "Function `_clone_scheduler_config` calls `SchedulerConfig`, `vars`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 234, "end_line": 238, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L234-L238", "decorators": [], "calls": [ "SchedulerConfig", "vars" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "SchedulerConfig(**vars(config))" ], "awaits": false, "yields": false }, { "name": "_parse_memory_budget_bytes", "qualname": "_parse_memory_budget_bytes", "full_name": "vllm_mlx.model_registry._parse_memory_budget_bytes", "kind": "function", "signature": "def _parse_memory_budget_bytes(value: Any) -> int", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Parse a memory budget from bytes, MB, or GB.", "summary": "Parse a memory budget from bytes, MB, or GB.", "implementation": "Function `_parse_memory_budget_bytes` calls `ValueError`, `isinstance`, `int`, `float`; can raise `ValueError`, `TypeError`; has 5 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 241, "end_line": 256, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L241-L256", "decorators": [], "calls": [ "ValueError", "isinstance", "int", "float", "value.strip().lower", "value.strip", "raw.endswith", "TypeError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError", "TypeError" ], "return_expressions": [ "int(float(value) * 1024 ** 3)", "int(float(raw[:-2]) * 1024 ** 3)", "int(float(raw[:-2]) * 1024 ** 2)", "int(float(raw[:-1]))", "int(float(raw) * 1024 ** 3)" ], "awaits": false, "yields": false }, { "name": "_safe_available_memory_bytes", "qualname": "_safe_available_memory_bytes", "full_name": "vllm_mlx.model_registry._safe_available_memory_bytes", "kind": "function", "signature": "def _safe_available_memory_bytes() -> int", "parameters": [], "return_annotation": "int", "docstring": "Best-effort available system memory.", "summary": "Best-effort available system memory.", "implementation": "Function `_safe_available_memory_bytes` calls `int`, `psutil.virtual_memory`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 259, "end_line": 263, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L259-L263", "decorators": [], "calls": [ "int", "psutil.virtual_memory" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0", "int(psutil.virtual_memory().available)" ], "awaits": false, "yields": false }, { "name": "_device_working_set_bytes", "qualname": "_device_working_set_bytes", "full_name": "vllm_mlx.model_registry._device_working_set_bytes", "kind": "function", "signature": "def _device_working_set_bytes() -> int | None", "parameters": [], "return_annotation": "int | None", "docstring": "Best-effort Metal recommended working-set size, or None when unavailable.", "summary": "Best-effort Metal recommended working-set size, or None when unavailable.", "implementation": "Function `_device_working_set_bytes` calls `mx.metal.is_available`, `mx.device_info`, `info.get`, `int`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 266, "end_line": 282, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L266-L282", "decorators": [], "calls": [ "mx.metal.is_available", "mx.device_info", "info.get", "int", "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "working_set or None" ], "awaits": false, "yields": false }, { "name": "MemoryBudgetReport", "qualname": "MemoryBudgetReport", "full_name": "vllm_mlx.model_registry.MemoryBudgetReport", "kind": "class", "signature": "class MemoryBudgetReport", "parameters": [ { "name": "budget_bytes", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "device_working_set_bytes", "kind": "field", "annotation": "int | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "gpu_memory_utilization", "kind": "field", "annotation": "float | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "gpu_memory_utilization_source", "kind": "field", "annotation": "str | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "per_engine_cache_limit_bytes", "kind": "field", "annotation": "int | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "per_engine_cache_percent", "kind": "field", "annotation": "float | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "continuous_batching_entries", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_entries", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "MemoryBudgetReport", "docstring": "Reconciliation of the manager weight budget with the Metal ceiling.\n\nThe manager budget counts model *weights* only, and the Metal allocation\nceiling (``gpu_memory_utilization`` x device working set) is process-wide.\nThose two are directly comparable, so a budget above the ceiling is a\ndeterministic conflict: the manager will keep models resident that MLX\ncannot allocate, and the load fails instead of evicting.\n\nThe prefix-cache limit is deliberately *not* folded into that comparison.\n``cache_memory_mb`` is a per-engine maximum — it is cloned into each\nresident continuous-batching engine and allocated lazily, and simple-mode\nentries never receive it at all — so it is neither a single process-wide\nreservation nor a bound that can be subtracted once. It is reported\nalongside the ceiling instead, with its own conflict check.", "summary": "Reconciliation of the manager weight budget with the Metal ceiling.", "implementation": "Class `MemoryBudgetReport` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 286, "end_line": 339, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L286-L339", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "allocation_ceiling_bytes", "qualname": "MemoryBudgetReport.allocation_ceiling_bytes", "full_name": "vllm_mlx.model_registry.MemoryBudgetReport.allocation_ceiling_bytes", "kind": "method", "signature": "def allocation_ceiling_bytes(self) -> int | None", "parameters": [], "return_annotation": "int | None", "docstring": "Metal soft allocation limit that will be installed at engine start.\n\n``None`` when no ceiling can be attributed: either MLX cannot report a\ndevice working set, or no entry will install one (only ``BatchedEngine``\ncalls ``mx.set_memory_limit``).", "summary": "Metal soft allocation limit that will be installed at engine start.", "implementation": "Method `MemoryBudgetReport.allocation_ceiling_bytes` calls `int`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 313, "end_line": 322, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L313-L322", "decorators": [ "property" ], "calls": [ "int" ], "state_reads": [ "self.device_working_set_bytes", "self.gpu_memory_utilization" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "int(self.device_working_set_bytes * self.gpu_memory_utilization)" ], "awaits": false, "yields": false }, { "name": "exceeds_ceiling", "qualname": "MemoryBudgetReport.exceeds_ceiling", "full_name": "vllm_mlx.model_registry.MemoryBudgetReport.exceeds_ceiling", "kind": "method", "signature": "def exceeds_ceiling(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "True when the weights budget alone cannot fit under the ceiling.\n\nBoth sides are process-wide totals, so this is the deterministic check.", "summary": "True when the weights budget alone cannot fit under the ceiling.", "implementation": "Method `MemoryBudgetReport.exceeds_ceiling` returns `ceiling is not None and self.budget_bytes > ceiling`.", "documented": true, "public": true, "addressable": true, "line": 325, "end_line": 331, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L325-L331", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.allocation_ceiling_bytes", "self.budget_bytes" ], "state_writes": [], "raises": [], "return_expressions": [ "ceiling is not None and self.budget_bytes > ceiling" ], "awaits": false, "yields": false }, { "name": "cache_limit_exceeds_ceiling", "qualname": "MemoryBudgetReport.cache_limit_exceeds_ceiling", "full_name": "vllm_mlx.model_registry.MemoryBudgetReport.cache_limit_exceeds_ceiling", "kind": "method", "signature": "def cache_limit_exceeds_ceiling(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "True when one engine's prefix cache could alone fill the ceiling.", "summary": "True when one engine's prefix cache could alone fill the ceiling.", "implementation": "Method `MemoryBudgetReport.cache_limit_exceeds_ceiling` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 334, "end_line": 339, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L334-L339", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.allocation_ceiling_bytes", "self.per_engine_cache_limit_bytes" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "self.per_engine_cache_limit_bytes >= ceiling" ], "awaits": false, "yields": false }, { "name": "build_memory_budget_report", "qualname": "build_memory_budget_report", "full_name": "vllm_mlx.model_registry.build_memory_budget_report", "kind": "function", "signature": "def build_memory_budget_report(manager_config: RegistryManagerConfig, registry: dict[str, RegisteredModel], defaults: RegistryServeDefaults, *, device_working_set_bytes: int | None=None) -> MemoryBudgetReport", "parameters": [ { "name": "manager_config", "kind": "positional or keyword", "annotation": "RegistryManagerConfig", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "registry", "kind": "positional or keyword", "annotation": "dict[str, RegisteredModel]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "defaults", "kind": "positional or keyword", "annotation": "RegistryServeDefaults", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "device_working_set_bytes", "kind": "keyword-only", "annotation": "int | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "MemoryBudgetReport", "docstring": "Reconcile the manager weight budget against the Metal allocation ceiling.\n\nThe Metal limit is process-wide but is re-installed by every\n``BatchedEngine`` start, so the ceiling the manager has to live under is the\n*lowest* utilization among the entries that actually install one. Only\ncontinuous-batching entries qualify: ``SimpleEngine`` never calls\n``mx.set_memory_limit`` and is not even given a ``gpu_memory_utilization``.\nA registry with no continuous-batching entries therefore gets no attributed\nceiling rather than one derived from a value nothing installs.", "summary": "Reconcile the manager weight budget against the Metal allocation ceiling.", "implementation": "Function `build_memory_budget_report` calls `_device_working_set_bytes`, `sorted`, `candidates.append`, `len`; returns `MemoryBudgetReport(budget_bytes=manager_config.memory_budget_bytes, device_working_set_bytes=device_working_set_bytes, …`.", "documented": true, "public": true, "addressable": true, "line": 342, "end_line": 421, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L342-L421", "decorators": [], "calls": [ "_device_working_set_bytes", "sorted", "candidates.append", "len", "min", "getattr", "int", "float", "MemoryBudgetReport" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "MemoryBudgetReport(budget_bytes=manager_config.memory_budget_bytes, device_working_set_bytes=device_working_set_bytes, …" ], "awaits": false, "yields": false }, { "name": "log_memory_budget_report", "qualname": "log_memory_budget_report", "full_name": "vllm_mlx.model_registry.log_memory_budget_report", "kind": "function", "signature": "def log_memory_budget_report(report: MemoryBudgetReport) -> None", "parameters": [ { "name": "report", "kind": "positional or keyword", "annotation": "MemoryBudgetReport", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Log the budget/ceiling reconciliation, warning when they conflict.", "summary": "Log the budget/ceiling reconciliation, warning when they conflict.", "implementation": "Function `log_memory_budget_report` calls `logger.info`, `logger.warning`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 424, "end_line": 502, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L424-L502", "decorators": [], "calls": [ "logger.info", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_estimate_model_bytes_from_source", "qualname": "_estimate_model_bytes_from_source", "full_name": "vllm_mlx.model_registry._estimate_model_bytes_from_source", "kind": "function", "signature": "def _estimate_model_bytes_from_source(source: str) -> int", "parameters": [ { "name": "source", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Estimate model footprint from local artifact size when possible.", "summary": "Estimate model footprint from local artifact size when possible.", "implementation": "Function `_estimate_model_bytes_from_source` calls `Path`, `path.exists`, `path.is_file`, `path.stat`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 505, "end_line": 521, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L505-L521", "decorators": [], "calls": [ "Path", "path.exists", "path.is_file", "path.stat", "path.rglob", "fp.stat" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0", "path.stat().st_size if path.suffix in {'.safetensors', '.gguf'} else 0", "total" ], "awaits": false, "yields": false }, { "name": "load_registry_config", "qualname": "load_registry_config", "full_name": "vllm_mlx.model_registry.load_registry_config", "kind": "function", "signature": "def load_registry_config(config_path: str | os.PathLike[str], defaults: RegistryServeDefaults) -> tuple[RegistryManagerConfig, dict[str, RegisteredModel]]", "parameters": [ { "name": "config_path", "kind": "positional or keyword", "annotation": "str | os.PathLike[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "defaults", "kind": "positional or keyword", "annotation": "RegistryServeDefaults", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[RegistryManagerConfig, dict[str, RegisteredModel]]", "docstring": "Load and validate the models registry YAML file.", "summary": "Load and validate the models registry YAML file.", "implementation": "Function `load_registry_config` calls `yaml.safe_load`, `Path(config_path).read_text`, `Path`, `raw.get`; can raise `ValueError`; returns `(manager, registry)`.", "documented": true, "public": true, "addressable": true, "line": 524, "end_line": 621, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L524-L621", "decorators": [], "calls": [ "yaml.safe_load", "Path(config_path).read_text", "Path", "raw.get", "isinstance", "ValueError", "manager_raw.get", "ContentionPolicy", "policy_raw.get", "float", "RegistryManagerConfig", "_parse_memory_budget_bytes", "item.get", "int", "math.isfinite", "RegisteredModel", "str", "bool" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "(manager, registry)" ], "awaits": false, "yields": false }, { "name": "ModelManager", "qualname": "ModelManager", "full_name": "vllm_mlx.model_registry.ModelManager", "kind": "class", "signature": "class ModelManager", "parameters": [ { "name": "manager_config", "kind": "positional or keyword", "annotation": "RegistryManagerConfig", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "registry", "kind": "positional or keyword", "annotation": "dict[str, RegisteredModel]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "defaults", "kind": "positional or keyword", "annotation": "RegistryServeDefaults", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine_factory", "kind": "keyword-only", "annotation": "EngineFactory | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "ModelManager", "docstring": "Registry-backed model manager with lazy load and memory-budget eviction.", "summary": "Registry-backed model manager with lazy load and memory-budget eviction.", "implementation": "Class `ModelManager` declares 27 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 624, "end_line": 1201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L624-L1201", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ModelManager.__init__", "full_name": "vllm_mlx.model_registry.ModelManager.__init__", "kind": "method", "signature": "def __init__(self, manager_config: RegistryManagerConfig, registry: dict[str, RegisteredModel], defaults: RegistryServeDefaults, *, engine_factory: EngineFactory | None=None) -> None", "parameters": [ { "name": "manager_config", "kind": "positional or keyword", "annotation": "RegistryManagerConfig", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "registry", "kind": "positional or keyword", "annotation": "dict[str, RegisteredModel]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "defaults", "kind": "positional or keyword", "annotation": "RegistryServeDefaults", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine_factory", "kind": "keyword-only", "annotation": "EngineFactory | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ModelManager.__init__` updates `self._config`, `self._registry`, `self._defaults`, `self._engine_factory`; calls `asyncio.Condition`.", "implementation": "Method `ModelManager.__init__` updates `self._config`, `self._registry`, `self._defaults`, `self._engine_factory`; calls `asyncio.Condition`.", "documented": false, "public": false, "addressable": true, "line": 627, "end_line": 643, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L627-L643", "decorators": [], "calls": [ "asyncio.Condition" ], "state_reads": [], "state_writes": [ "self._config", "self._registry", "self._defaults", "self._engine_factory", "self._loaded", "self._loading", "self._unloading", "self._condition", "self._shutting_down" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "memory_budget_bytes", "qualname": "ModelManager.memory_budget_bytes", "full_name": "vllm_mlx.model_registry.ModelManager.memory_budget_bytes", "kind": "method", "signature": "def memory_budget_bytes(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return the registry's configured resident-model memory budget.", "summary": "Return the registry's configured resident-model memory budget.", "implementation": "Method `ModelManager.memory_budget_bytes` returns `self._config.memory_budget_bytes`.", "documented": true, "public": true, "addressable": true, "line": 646, "end_line": 649, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L646-L649", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._config.memory_budget_bytes", "self._config" ], "state_writes": [], "raises": [], "return_expressions": [ "self._config.memory_budget_bytes" ], "awaits": false, "yields": false }, { "name": "registered_model_names", "qualname": "ModelManager.registered_model_names", "full_name": "vllm_mlx.model_registry.ModelManager.registered_model_names", "kind": "method", "signature": "def registered_model_names(self) -> list[str]", "parameters": [], "return_annotation": "list[str]", "docstring": "Return sorted list of all registered model names.", "summary": "Return sorted list of all registered model names.", "implementation": "Method `ModelManager.registered_model_names` calls `sorted`, `self._registry.keys`; returns `sorted(self._registry.keys())`.", "documented": true, "public": true, "addressable": true, "line": 652, "end_line": 654, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L652-L654", "decorators": [ "property" ], "calls": [ "sorted", "self._registry.keys" ], "state_reads": [ "self._registry.keys", "self._registry" ], "state_writes": [], "raises": [], "return_expressions": [ "sorted(self._registry.keys())" ], "awaits": false, "yields": false }, { "name": "has_model", "qualname": "ModelManager.has_model", "full_name": "vllm_mlx.model_registry.ModelManager.has_model", "kind": "method", "signature": "def has_model(self, model_name: str) -> bool", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return whether a model name is present in the serving registry.", "summary": "Return whether a model name is present in the serving registry.", "implementation": "Method `ModelManager.has_model` returns `model_name in self._registry`.", "documented": true, "public": true, "addressable": true, "line": 656, "end_line": 659, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L656-L659", "decorators": [], "calls": [], "state_reads": [ "self._registry" ], "state_writes": [], "raises": [], "return_expressions": [ "model_name in self._registry" ], "awaits": false, "yields": false }, { "name": "list_models", "qualname": "ModelManager.list_models", "full_name": "vllm_mlx.model_registry.ModelManager.list_models", "kind": "method", "signature": "def list_models(self) -> list[dict[str, Any]]", "parameters": [], "return_annotation": "list[dict[str, Any]]", "docstring": "Return registry state for /v1/models.", "summary": "Return registry state for /v1/models.", "implementation": "Method `ModelManager.list_models` calls `self._registry.items`, `self._loaded.get`, `self._unloading.get`, `self._loading.get`; returns `data`.", "documented": true, "public": true, "addressable": true, "line": 661, "end_line": 699, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L661-L699", "decorators": [], "calls": [ "self._registry.items", "self._loaded.get", "self._unloading.get", "self._loading.get", "self._resolve_estimated_bytes", "data.append", "round" ], "state_reads": [ "self._registry.items", "self._registry", "self._loaded.get", "self._loaded", "self._unloading.get", "self._unloading", "self._loading.get", "self._loading", "self._resolve_estimated_bytes" ], "state_writes": [], "raises": [], "return_expressions": [ "data" ], "awaits": false, "yields": false }, { "name": "preload", "qualname": "ModelManager.preload", "full_name": "vllm_mlx.model_registry.ModelManager.preload", "kind": "method", "signature": "async def preload(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Preload any entries marked preload=true.", "summary": "Preload any entries marked preload=true.", "implementation": "Method `ModelManager.preload` calls `self._registry.values`, `self.acquire`, `lease.release`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 701, "end_line": 706, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L701-L706", "decorators": [], "calls": [ "self._registry.values", "self.acquire", "lease.release" ], "state_reads": [ "self._registry.values", "self._registry", "self.acquire" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "shutdown", "qualname": "ModelManager.shutdown", "full_name": "vllm_mlx.model_registry.ModelManager.shutdown", "kind": "method", "signature": "async def shutdown(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Stop and unload every loaded engine.", "summary": "Stop and unload every loaded engine.", "implementation": "Method `ModelManager.shutdown` updates `self._shutting_down`; calls `set`, `self._loading.values`, `self._loaded.values`, `cancel_tasks.update`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 708, "end_line": 739, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L708-L739", "decorators": [], "calls": [ "set", "self._loading.values", "self._loaded.values", "cancel_tasks.update", "list", "self._loaded.keys", "unloads.append", "self._begin_unload_locked", "self._condition.notify_all", "task.cancel", "self._run_unloads", "asyncio.gather", "remaining.append" ], "state_reads": [ "self._condition", "self._loading.values", "self._loading", "self._loaded.values", "self._loaded", "self._loaded.keys", "self._begin_unload_locked", "self._condition.notify_all", "self._run_unloads" ], "state_writes": [ "self._shutting_down" ], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "acquire", "qualname": "ModelManager.acquire", "full_name": "vllm_mlx.model_registry.ModelManager.acquire", "kind": "method", "signature": "async def acquire(self, model_name: str) -> ModelLease", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "ModelLease", "docstring": "Acquire a lease for a configured model.", "summary": "Acquire a lease for a configured model.", "implementation": "Method `ModelManager.acquire` calls `KeyError`, `time.monotonic`, `set`, `RuntimeError`; awaits asynchronous work; can raise `KeyError`, `RuntimeError`; returns `claimed`.", "documented": true, "public": true, "addressable": true, "line": 741, "end_line": 819, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L741-L819", "decorators": [], "calls": [ "KeyError", "time.monotonic", "set", "RuntimeError", "self._claim_loaded_locked", "self._loading.get", "self._remaining_wait_timeout", "self._resolve_estimated_bytes", "self._collect_idle_unloads_locked", "self._can_reserve_locked", "self._reserve_load_locked", "self._maybe_preempt_locked", "self._should_wait_locked", "self._run_unloads", "task.cancel", "self._wait_for_change", "self._execute_load" ], "state_reads": [ "self._registry", "self._condition", "self._shutting_down", "self._claim_loaded_locked", "self._loading.get", "self._loading", "self._unloading", "self._remaining_wait_timeout", "self._resolve_estimated_bytes", "self._collect_idle_unloads_locked", "self._can_reserve_locked", "self._reserve_load_locked", "self._maybe_preempt_locked", "self._should_wait_locked", "self._config.memory_budget_bytes", "self._config", "self._run_unloads", "self._wait_for_change", "self._execute_load" ], "state_writes": [], "raises": [ "KeyError", "RuntimeError" ], "return_expressions": [ "claimed" ], "awaits": true, "yields": false }, { "name": "release", "qualname": "ModelManager.release", "full_name": "vllm_mlx.model_registry.ModelManager.release", "kind": "method", "signature": "async def release(self, model_name: str) -> None", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Release a previously acquired model lease.", "summary": "Release a previously acquired model lease.", "implementation": "Method `ModelManager.release` calls `self._loaded.get`, `max`, `time.time`, `asyncio.current_task`; awaits asynchronous work; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 821, "end_line": 842, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L821-L842", "decorators": [], "calls": [ "self._loaded.get", "max", "time.time", "asyncio.current_task", "loaded.active_tasks.discard", "self._begin_unload_locked", "self._condition.notify_all", "self._run_unloads" ], "state_reads": [ "self._condition", "self._loaded.get", "self._loaded", "self._begin_unload_locked", "self._condition.notify_all", "self._run_unloads" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_claim_loaded_locked", "qualname": "ModelManager._claim_loaded_locked", "full_name": "vllm_mlx.model_registry.ModelManager._claim_loaded_locked", "kind": "method", "signature": "def _claim_loaded_locked(self, model_name: str, *, loaded_override: LoadedModel | None=None) -> ModelLease | None", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "loaded_override", "kind": "keyword-only", "annotation": "LoadedModel | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "ModelLease | None", "docstring": "", "summary": "Method `ModelManager._claim_loaded_locked` calls `self._loaded.get`, `time.time`, `asyncio.current_task`, `loaded.active_tasks.add`; has 2 explicit return paths.", "implementation": "Method `ModelManager._claim_loaded_locked` calls `self._loaded.get`, `time.time`, `asyncio.current_task`, `loaded.active_tasks.add`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 844, "end_line": 874, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L844-L874", "decorators": [], "calls": [ "self._loaded.get", "time.time", "asyncio.current_task", "loaded.active_tasks.add", "ModelLease" ], "state_reads": [ "self._loaded.get", "self._loaded" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "ModelLease(manager=self, model_name=model_name, engine=loaded.engine, release_cb=_release)" ], "awaits": false, "yields": false }, { "name": "_release", "qualname": "ModelManager._claim_loaded_locked._release", "full_name": "vllm_mlx.model_registry.ModelManager._claim_loaded_locked._release", "kind": "nested function", "signature": "async def _release() -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Nested Function `ModelManager._claim_loaded_locked._release` calls `self.release`; awaits asynchronous work.", "implementation": "Nested Function `ModelManager._claim_loaded_locked._release` calls `self.release`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 866, "end_line": 867, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L866-L867", "decorators": [], "calls": [ "self.release" ], "state_reads": [ "self.release" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_execute_load", "qualname": "ModelManager._execute_load", "full_name": "vllm_mlx.model_registry.ModelManager._execute_load", "kind": "method", "signature": "async def _execute_load(self, pending: PendingLoad) -> LoadedModel", "parameters": [ { "name": "pending", "kind": "positional or keyword", "annotation": "PendingLoad", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "LoadedModel", "docstring": "Instantiate a reserved model load outside the manager lock.", "summary": "Instantiate a reserved model load outside the manager lock.", "implementation": "Method `ModelManager._execute_load` calls `self._resolve_source`, `self._instantiate_model`, `self._loading.pop`, `current.future.done`; awaits asynchronous work; can raise `RuntimeError`; returns `loaded`.", "documented": true, "public": false, "addressable": true, "line": 876, "end_line": 913, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L876-L913", "decorators": [], "calls": [ "self._resolve_source", "self._instantiate_model", "self._loading.pop", "current.future.done", "current.future.set_exception", "self._condition.notify_all", "RuntimeError", "current.future.set_result", "unload_after_load.engine.stop" ], "state_reads": [ "self._registry", "self._resolve_source", "self._instantiate_model", "self._condition", "self._loading.pop", "self._loading", "self._condition.notify_all", "self._shutting_down", "self._loaded" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "loaded" ], "awaits": true, "yields": false }, { "name": "_wait_for_change", "qualname": "ModelManager._wait_for_change", "full_name": "vllm_mlx.model_registry.ModelManager._wait_for_change", "kind": "method", "signature": "async def _wait_for_change(self, timeout: float | None) -> None", "parameters": [ { "name": "timeout", "kind": "positional or keyword", "annotation": "float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ModelManager._wait_for_change` calls `self._condition.wait`, `RuntimeError`, `asyncio.wait_for`; awaits asynchronous work; can raise `RuntimeError`; returns `None`.", "implementation": "Method `ModelManager._wait_for_change` calls `self._condition.wait`, `RuntimeError`, `asyncio.wait_for`; awaits asynchronous work; can raise `RuntimeError`; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 915, "end_line": 922, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L915-L922", "decorators": [], "calls": [ "self._condition.wait", "RuntimeError", "asyncio.wait_for" ], "state_reads": [ "self._condition", "self._condition.wait" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_run_unloads", "qualname": "ModelManager._run_unloads", "full_name": "vllm_mlx.model_registry.ModelManager._run_unloads", "kind": "method", "signature": "async def _run_unloads(self, unloads: list[LoadedModel]) -> None", "parameters": [ { "name": "unloads", "kind": "positional or keyword", "annotation": "list[LoadedModel]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `ModelManager._run_unloads` calls `loaded.engine.stop`, `self._unloading.pop`, `self._condition.notify_all`; awaits asynchronous work.", "implementation": "Method `ModelManager._run_unloads` calls `loaded.engine.stop`, `self._unloading.pop`, `self._condition.notify_all`; awaits asynchronous work.", "documented": false, "public": false, "addressable": true, "line": 924, "end_line": 931, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L924-L931", "decorators": [], "calls": [ "loaded.engine.stop", "self._unloading.pop", "self._condition.notify_all" ], "state_reads": [ "self._condition", "self._unloading.pop", "self._unloading", "self._condition.notify_all" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_reserve_load_locked", "qualname": "ModelManager._reserve_load_locked", "full_name": "vllm_mlx.model_registry.ModelManager._reserve_load_locked", "kind": "method", "signature": "def _reserve_load_locked(self, model_name: str, required_bytes: int) -> PendingLoad", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "required_bytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "PendingLoad", "docstring": "", "summary": "Method `ModelManager._reserve_load_locked` calls `asyncio.get_running_loop().create_future`, `asyncio.get_running_loop`, `PendingLoad`; returns `pending`.", "implementation": "Method `ModelManager._reserve_load_locked` calls `asyncio.get_running_loop().create_future`, `asyncio.get_running_loop`, `PendingLoad`; returns `pending`.", "documented": false, "public": false, "addressable": true, "line": 933, "end_line": 941, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L933-L941", "decorators": [], "calls": [ "asyncio.get_running_loop().create_future", "asyncio.get_running_loop", "PendingLoad" ], "state_reads": [ "self._loading" ], "state_writes": [], "raises": [], "return_expressions": [ "pending" ], "awaits": false, "yields": false }, { "name": "_begin_unload_locked", "qualname": "ModelManager._begin_unload_locked", "full_name": "vllm_mlx.model_registry.ModelManager._begin_unload_locked", "kind": "method", "signature": "def _begin_unload_locked(self, model_name: str) -> LoadedModel", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "LoadedModel", "docstring": "", "summary": "Method `ModelManager._begin_unload_locked` calls `self._loaded.pop`; returns `loaded`.", "implementation": "Method `ModelManager._begin_unload_locked` calls `self._loaded.pop`; returns `loaded`.", "documented": false, "public": false, "addressable": true, "line": 943, "end_line": 946, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L943-L946", "decorators": [], "calls": [ "self._loaded.pop" ], "state_reads": [ "self._loaded.pop", "self._loaded", "self._unloading" ], "state_writes": [], "raises": [], "return_expressions": [ "loaded" ], "awaits": false, "yields": false }, { "name": "_collect_idle_unloads_locked", "qualname": "ModelManager._collect_idle_unloads_locked", "full_name": "vllm_mlx.model_registry.ModelManager._collect_idle_unloads_locked", "kind": "method", "signature": "def _collect_idle_unloads_locked(self, requested_model: str, required_bytes: int) -> list[LoadedModel]", "parameters": [ { "name": "requested_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "required_bytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[LoadedModel]", "docstring": "", "summary": "Method `ModelManager._collect_idle_unloads_locked` calls `self._committed_bytes_locked`, `sorted`, `self._loaded.items`, `selected.append`; returns `selected`.", "implementation": "Method `ModelManager._collect_idle_unloads_locked` calls `self._committed_bytes_locked`, `sorted`, `self._loaded.items`, `selected.append`; returns `selected`.", "documented": false, "public": false, "addressable": true, "line": 948, "end_line": 968, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L948-L968", "decorators": [], "calls": [ "self._committed_bytes_locked", "sorted", "self._loaded.items", "selected.append", "self._begin_unload_locked" ], "state_reads": [ "self._committed_bytes_locked", "self._loaded.items", "self._loaded", "self._config.memory_budget_bytes", "self._config", "self._begin_unload_locked" ], "state_writes": [], "raises": [], "return_expressions": [ "selected" ], "awaits": false, "yields": false }, { "name": "_maybe_preempt_locked", "qualname": "ModelManager._maybe_preempt_locked", "full_name": "vllm_mlx.model_registry.ModelManager._maybe_preempt_locked", "kind": "method", "signature": "def _maybe_preempt_locked(self, *, model_name: str, required_bytes: int, start: float) -> set[asyncio.Task[Any]]", "parameters": [ { "name": "model_name", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "required_bytes", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "start", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "set[asyncio.Task[Any]]", "docstring": "", "summary": "Method `ModelManager._maybe_preempt_locked` calls `self._should_preempt_locked`, `set`, `self._committed_bytes_locked`, `sorted`; has 2 explicit return paths.", "implementation": "Method `ModelManager._maybe_preempt_locked` calls `self._should_preempt_locked`, `set`, `self._committed_bytes_locked`, `sorted`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 970, "end_line": 1003, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L970-L1003", "decorators": [], "calls": [ "self._should_preempt_locked", "set", "self._committed_bytes_locked", "sorted", "self._loaded.items", "cancel_tasks.update", "self._condition.notify_all" ], "state_reads": [ "self._should_preempt_locked", "self._committed_bytes_locked", "self._loaded.items", "self._loaded", "self._config.memory_budget_bytes", "self._config", "self._condition.notify_all", "self._condition" ], "state_writes": [], "raises": [], "return_expressions": [ "set()", "cancel_tasks" ], "awaits": false, "yields": false }, { "name": "_should_wait_locked", "qualname": "ModelManager._should_wait_locked", "full_name": "vllm_mlx.model_registry.ModelManager._should_wait_locked", "kind": "method", "signature": "def _should_wait_locked(self, start: float) -> bool", "parameters": [ { "name": "start", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "", "summary": "Method `ModelManager._should_wait_locked` calls `self._remaining_wait_timeout`; has 2 explicit return paths.", "implementation": "Method `ModelManager._should_wait_locked` calls `self._remaining_wait_timeout`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1005, "end_line": 1010, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1005-L1010", "decorators": [], "calls": [ "self._remaining_wait_timeout" ], "state_reads": [ "self._config.policy.strategy", "self._config.policy", "self._config", "self._remaining_wait_timeout" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "timeout is None or timeout > 0" ], "awaits": false, "yields": false }, { "name": "_should_preempt_locked", "qualname": "ModelManager._should_preempt_locked", "full_name": "vllm_mlx.model_registry.ModelManager._should_preempt_locked", "kind": "method", "signature": "def _should_preempt_locked(self, start: float) -> bool", "parameters": [ { "name": "start", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "", "summary": "Method `ModelManager._should_preempt_locked` calls `time.monotonic`; has 3 explicit return paths.", "implementation": "Method `ModelManager._should_preempt_locked` calls `time.monotonic`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1012, "end_line": 1020, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1012-L1020", "decorators": [], "calls": [ "time.monotonic" ], "state_reads": [ "self._config.policy", "self._config" ], "state_writes": [], "raises": [], "return_expressions": [ "True", "False", "elapsed >= trigger" ], "awaits": false, "yields": false }, { "name": "_remaining_wait_timeout", "qualname": "ModelManager._remaining_wait_timeout", "full_name": "vllm_mlx.model_registry.ModelManager._remaining_wait_timeout", "kind": "method", "signature": "def _remaining_wait_timeout(self, start: float) -> float | None", "parameters": [ { "name": "start", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float | None", "docstring": "", "summary": "Method `ModelManager._remaining_wait_timeout` calls `max`, `time.monotonic`; has 2 explicit return paths.", "implementation": "Method `ModelManager._remaining_wait_timeout` calls `max`, `time.monotonic`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1022, "end_line": 1026, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1022-L1026", "decorators": [], "calls": [ "max", "time.monotonic" ], "state_reads": [ "self._config.policy.wait_timeout_s", "self._config.policy", "self._config" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "max(timeout - (time.monotonic() - start), 0.0)" ], "awaits": false, "yields": false }, { "name": "_can_reserve_locked", "qualname": "ModelManager._can_reserve_locked", "full_name": "vllm_mlx.model_registry.ModelManager._can_reserve_locked", "kind": "method", "signature": "def _can_reserve_locked(self, required_bytes: int) -> bool", "parameters": [ { "name": "required_bytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "", "summary": "Method `ModelManager._can_reserve_locked` calls `self._committed_bytes_locked`; returns `self._committed_bytes_locked() + required_bytes <= self._config.memory_budget_bytes`.", "implementation": "Method `ModelManager._can_reserve_locked` calls `self._committed_bytes_locked`; returns `self._committed_bytes_locked() + required_bytes <= self._config.memory_budget_bytes`.", "documented": false, "public": false, "addressable": true, "line": 1028, "end_line": 1032, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1028-L1032", "decorators": [], "calls": [ "self._committed_bytes_locked" ], "state_reads": [ "self._committed_bytes_locked", "self._config.memory_budget_bytes", "self._config" ], "state_writes": [], "raises": [], "return_expressions": [ "self._committed_bytes_locked() + required_bytes <= self._config.memory_budget_bytes" ], "awaits": false, "yields": false }, { "name": "_committed_bytes_locked", "qualname": "ModelManager._committed_bytes_locked", "full_name": "vllm_mlx.model_registry.ModelManager._committed_bytes_locked", "kind": "method", "signature": "def _committed_bytes_locked(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "", "summary": "Method `ModelManager._committed_bytes_locked` calls `sum`, `self._loaded.values`, `self._loading.values`, `self._unloading.values`; returns `loaded_bytes + loading_bytes + unloading_bytes`.", "implementation": "Method `ModelManager._committed_bytes_locked` calls `sum`, `self._loaded.values`, `self._loading.values`, `self._unloading.values`; returns `loaded_bytes + loading_bytes + unloading_bytes`.", "documented": false, "public": false, "addressable": true, "line": 1034, "end_line": 1042, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1034-L1042", "decorators": [], "calls": [ "sum", "self._loaded.values", "self._loading.values", "self._unloading.values" ], "state_reads": [ "self._loaded.values", "self._loaded", "self._loading.values", "self._loading", "self._unloading.values", "self._unloading" ], "state_writes": [], "raises": [], "return_expressions": [ "loaded_bytes + loading_bytes + unloading_bytes" ], "awaits": false, "yields": false }, { "name": "_instantiate_model", "qualname": "ModelManager._instantiate_model", "full_name": "vllm_mlx.model_registry.ModelManager._instantiate_model", "kind": "method", "signature": "async def _instantiate_model(self, entry: RegisteredModel, resolved_source: str) -> LoadedModel", "parameters": [ { "name": "entry", "kind": "positional or keyword", "annotation": "RegisteredModel", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "resolved_source", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "LoadedModel", "docstring": "", "summary": "Method `ModelManager._instantiate_model` calls `self._resolve_model_config`, `self._engine_factory`, `BatchedEngine`, `SimpleEngine`; awaits asynchronous work; returns `LoadedModel(config=config, engine=engine)`.", "implementation": "Method `ModelManager._instantiate_model` calls `self._resolve_model_config`, `self._engine_factory`, `BatchedEngine`, `SimpleEngine`; awaits asynchronous work; returns `LoadedModel(config=config, engine=engine)`.", "documented": false, "public": false, "addressable": true, "line": 1044, "end_line": 1073, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1044-L1073", "decorators": [], "calls": [ "self._resolve_model_config", "self._engine_factory", "BatchedEngine", "SimpleEngine", "engine.start", "LoadedModel" ], "state_reads": [ "self._resolve_model_config", "self._engine_factory" ], "state_writes": [], "raises": [], "return_expressions": [ "LoadedModel(config=config, engine=engine)" ], "awaits": true, "yields": false }, { "name": "_resolve_source", "qualname": "ModelManager._resolve_source", "full_name": "vllm_mlx.model_registry.ModelManager._resolve_source", "kind": "method", "signature": "async def _resolve_source(self, entry: RegisteredModel) -> str", "parameters": [ { "name": "entry", "kind": "positional or keyword", "annotation": "RegisteredModel", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Method `ModelManager._resolve_source` calls `asyncio.to_thread`; awaits asynchronous work; returns `await asyncio.to_thread(self._resolve_source_sync, entry)`.", "implementation": "Method `ModelManager._resolve_source` calls `asyncio.to_thread`; awaits asynchronous work; returns `await asyncio.to_thread(self._resolve_source_sync, entry)`.", "documented": false, "public": false, "addressable": true, "line": 1075, "end_line": 1076, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1075-L1076", "decorators": [], "calls": [ "asyncio.to_thread" ], "state_reads": [ "self._resolve_source_sync" ], "state_writes": [], "raises": [], "return_expressions": [ "await asyncio.to_thread(self._resolve_source_sync, entry)" ], "awaits": true, "yields": false }, { "name": "_resolve_source_sync", "qualname": "ModelManager._resolve_source_sync", "full_name": "vllm_mlx.model_registry.ModelManager._resolve_source_sync", "kind": "method", "signature": "def _resolve_source_sync(self, entry: RegisteredModel) -> str", "parameters": [ { "name": "entry", "kind": "positional or keyword", "annotation": "RegisteredModel", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Method `ModelManager._resolve_source_sync` calls `Path(source).exists`, `Path`, `ensure_model_downloaded`, `is_mllm_model`; has 2 explicit return paths.", "implementation": "Method `ModelManager._resolve_source_sync` calls `Path(source).exists`, `Path`, `ensure_model_downloaded`, `is_mllm_model`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1078, "end_line": 1087, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1078-L1087", "decorators": [], "calls": [ "Path(source).exists", "Path", "ensure_model_downloaded", "is_mllm_model", "bool", "str" ], "state_reads": [ "self._defaults.download_config", "self._defaults" ], "state_writes": [], "raises": [], "return_expressions": [ "source", "str(downloaded)" ], "awaits": false, "yields": false }, { "name": "_resolve_estimated_bytes", "qualname": "ModelManager._resolve_estimated_bytes", "full_name": "vllm_mlx.model_registry.ModelManager._resolve_estimated_bytes", "kind": "method", "signature": "def _resolve_estimated_bytes(self, entry: RegisteredModel, resolved_source: str) -> int", "parameters": [ { "name": "entry", "kind": "positional or keyword", "annotation": "RegisteredModel", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "resolved_source", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "", "summary": "Method `ModelManager._resolve_estimated_bytes` calls `_estimate_model_bytes_from_source`, `Path`, `source_path.exists`, `ValueError`; can raise `ValueError`; has 3 explicit return paths.", "implementation": "Method `ModelManager._resolve_estimated_bytes` calls `_estimate_model_bytes_from_source`, `Path`, `source_path.exists`, `ValueError`; can raise `ValueError`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1089, "end_line": 1121, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1089-L1121", "decorators": [], "calls": [ "_estimate_model_bytes_from_source", "Path", "source_path.exists", "ValueError", "_safe_available_memory_bytes", "logger.warning", "max" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "entry.estimated_memory_bytes", "estimated", "max(available // 8, 1)" ], "awaits": false, "yields": false }, { "name": "_resolve_model_config", "qualname": "ModelManager._resolve_model_config", "full_name": "vllm_mlx.model_registry.ModelManager._resolve_model_config", "kind": "method", "signature": "def _resolve_model_config(self, entry: RegisteredModel, resolved_source: str) -> ResolvedModelConfig", "parameters": [ { "name": "entry", "kind": "positional or keyword", "annotation": "RegisteredModel", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "resolved_source", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "ResolvedModelConfig", "docstring": "", "summary": "Method `ModelManager._resolve_model_config` calls `_clone_scheduler_config`, `self._resolve_estimated_bytes`, `ResolvedModelConfig`; returns `ResolvedModelConfig(entry=entry, resolved_source=resolved_source, continuous_batching=continuous_batching, force_mllm=f…`.", "implementation": "Method `ModelManager._resolve_model_config` calls `_clone_scheduler_config`, `self._resolve_estimated_bytes`, `ResolvedModelConfig`; returns `ResolvedModelConfig(entry=entry, resolved_source=resolved_source, continuous_batching=continuous_batching, force_mllm=f…`.", "documented": false, "public": false, "addressable": true, "line": 1123, "end_line": 1201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_registry.py#L1123-L1201", "decorators": [], "calls": [ "_clone_scheduler_config", "self._resolve_estimated_bytes", "ResolvedModelConfig" ], "state_reads": [ "self._defaults.scheduler_config", "self._defaults", "self._defaults.continuous_batching", "self._defaults.force_mllm", "self._defaults.enable_mtp", "self._defaults.prefill_step_size", "self._defaults.specprefill_enabled", "self._defaults.specprefill_threshold", "self._defaults.specprefill_keep_pct", "self._defaults.specprefill_backbone_pct", "self._defaults.specprefill_draft_model", "self._defaults.stream_interval", "self._defaults.gpu_memory_utilization", "self._resolve_estimated_bytes" ], "state_writes": [], "raises": [], "return_expressions": [ "ResolvedModelConfig(entry=entry, resolved_source=resolved_source, continuous_batching=continuous_batching, force_mllm=f…" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.model_runner", "path": "vllm_mlx/model_runner.py", "page_path": "reference/api/vllm_mlx/model_runner.md", "docstring": "MLX Model Runner for vLLM.\n\nThis module implements the model runner that bridges vLLM's request\nhandling with mlx-lm's inference capabilities.\n\nIncludes low-level optimizations:\n- mx.compile() for kernel fusion\n- Memory bandwidth optimization\n- Prefill chunking for L2 cache efficiency", "summary": "MLX Model Runner for vLLM.", "line_count": 476, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L1-L476", "members": [ "logger", "SamplerOutput", "MLXModelRunnerOutput", "MLXModelRunner" ], "symbols": [ { "name": "SamplerOutput", "qualname": "SamplerOutput", "full_name": "vllm_mlx.model_runner.SamplerOutput", "kind": "class", "signature": "class SamplerOutput", "parameters": [ { "name": "token_ids", "kind": "field", "annotation": "list[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "logprobs", "kind": "field", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "SamplerOutput", "docstring": "Output from sampling.", "summary": "Output from sampling.", "implementation": "Class `SamplerOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 29, "end_line": 33, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L29-L33", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLXModelRunnerOutput", "qualname": "MLXModelRunnerOutput", "full_name": "vllm_mlx.model_runner.MLXModelRunnerOutput", "kind": "class", "signature": "class MLXModelRunnerOutput", "parameters": [ { "name": "req_id_to_token_ids", "kind": "field", "annotation": "dict[str, list[int]]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "req_id_to_logprobs", "kind": "field", "annotation": "dict[str, list[dict]] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "num_tokens_generated", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "generation_time_s", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "MLXModelRunnerOutput", "docstring": "Output from MLX model runner, compatible with vLLM's ModelRunnerOutput.", "summary": "Output from MLX model runner, compatible with vLLM's ModelRunnerOutput.", "implementation": "Class `MLXModelRunnerOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 37, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L37-L50", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLXModelRunner", "qualname": "MLXModelRunner", "full_name": "vllm_mlx.model_runner.MLXModelRunner", "kind": "class", "signature": "class MLXModelRunner", "parameters": [ { "name": "vllm_config", "kind": "positional or keyword", "annotation": "'VllmConfig'", "default": "", "required": true, "description": "vLLM configuration" }, { "name": "enable_optimizations", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether to enable low-level optimizations" } ], "return_annotation": "MLXModelRunner", "docstring": "Model runner that uses mlx-lm for inference.\n\nThis class handles:\n- Model loading via mlx-lm\n- Converting vLLM requests to mlx-lm format\n- Running inference and returning results in vLLM format\n- KV cache management (delegated to mlx-lm)\n\nOptimizations:\n- mx.compile() for kernel fusion (fuses multiple ops into single Metal kernel)\n- Memory optimization for bandwidth efficiency\n- Prefill chunking for L2 cache utilization", "summary": "Model runner that uses mlx-lm for inference.", "implementation": "Class `MLXModelRunner` declares 16 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 53, "end_line": 476, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L53-L476", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLXModelRunner.__init__", "full_name": "vllm_mlx.model_runner.MLXModelRunner.__init__", "kind": "method", "signature": "def __init__(self, vllm_config: 'VllmConfig', enable_optimizations: bool=True)", "parameters": [ { "name": "vllm_config", "kind": "positional or keyword", "annotation": "'VllmConfig'", "default": "", "required": true, "description": "vLLM configuration" }, { "name": "enable_optimizations", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether to enable low-level optimizations" } ], "return_annotation": "not annotated", "docstring": "Initialize MLX model runner.\n\nArgs:\n vllm_config: vLLM configuration\n enable_optimizations: Whether to enable low-level optimizations", "summary": "Initialize MLX model runner.", "implementation": "Method `MLXModelRunner.__init__` updates `self.vllm_config`, `self.model_config`, `self.cache_config`, `self.scheduler_config`; calls `logger.info`.", "documented": true, "public": false, "addressable": true, "line": 69, "end_line": 104, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L69-L104", "decorators": [], "calls": [ "logger.info" ], "state_reads": [ "self.model_config.model", "self.model_config" ], "state_writes": [ "self.vllm_config", "self.model_config", "self.cache_config", "self.scheduler_config", "self.model", "self.tokenizer", "self._loaded", "self._sampler", "self._prompt_cache", "self._num_cache_blocks", "self._enable_optimizations", "self._compiled_forward", "self._hardware_info" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load_model", "qualname": "MLXModelRunner.load_model", "full_name": "vllm_mlx.model_runner.MLXModelRunner.load_model", "kind": "method", "signature": "def load_model(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load model using mlx-lm with optimizations.", "summary": "Load model using mlx-lm with optimizations.", "implementation": "Method `MLXModelRunner.load_model` updates `self.model`, `self.tokenizer`, `self._loaded`; calls `logger.info`, `time.time`, `load`, `self._create_default_sampler`; can raise `ImportError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 106, "end_line": 145, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L106-L145", "decorators": [], "calls": [ "logger.info", "time.time", "load", "self._create_default_sampler", "self._apply_optimizations", "ImportError", "logger.error" ], "state_reads": [ "self._loaded", "self.model_config.model", "self.model_config", "self.model_config.trust_remote_code", "self._create_default_sampler", "self._enable_optimizations", "self._apply_optimizations" ], "state_writes": [ "self.model", "self.tokenizer", "self._loaded" ], "raises": [ "ImportError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_apply_optimizations", "qualname": "MLXModelRunner._apply_optimizations", "full_name": "vllm_mlx.model_runner.MLXModelRunner._apply_optimizations", "kind": "method", "signature": "def _apply_optimizations(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Apply low-level optimizations for maximum performance.", "summary": "Apply low-level optimizations for maximum performance.", "implementation": "Method `MLXModelRunner._apply_optimizations` updates `self._hardware_info`; calls `detect_hardware`, `logger.info`, `configure_memory_optimization`, `self._setup_compiled_forward`.", "documented": true, "public": false, "addressable": true, "line": 147, "end_line": 168, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L147-L168", "decorators": [], "calls": [ "detect_hardware", "logger.info", "configure_memory_optimization", "self._setup_compiled_forward", "logger.warning" ], "state_reads": [ "self._hardware_info.chip_name", "self._hardware_info", "self._hardware_info.total_memory_gb", "self._hardware_info.memory_bandwidth_gbs", "self._setup_compiled_forward" ], "state_writes": [ "self._hardware_info" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_setup_compiled_forward", "qualname": "MLXModelRunner._setup_compiled_forward", "full_name": "vllm_mlx.model_runner.MLXModelRunner._setup_compiled_forward", "kind": "method", "signature": "def _setup_compiled_forward(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Setup compiled forward pass using mx.compile() for kernel fusion.\n\nThis fuses multiple operations into single Metal kernels,\nreducing kernel launch overhead and improving throughput.", "summary": "Setup compiled forward pass using mx.compile() for kernel fusion.", "implementation": "Method `MLXModelRunner._setup_compiled_forward` updates `self._compiled_forward`; calls `hasattr`, `mx.compile`, `logger.info`, `logger.warning`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 170, "end_line": 193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L170-L193", "decorators": [], "calls": [ "hasattr", "mx.compile", "logger.info", "logger.warning" ], "state_reads": [ "self.model", "self.model.__call__" ], "state_writes": [ "self._compiled_forward" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_create_default_sampler", "qualname": "MLXModelRunner._create_default_sampler", "full_name": "vllm_mlx.model_runner.MLXModelRunner._create_default_sampler", "kind": "method", "signature": "def _create_default_sampler(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Create default sampler for generation.", "summary": "Create default sampler for generation.", "implementation": "Method `MLXModelRunner._create_default_sampler` updates `self._sampler`; calls `make_sampler`, `logger.warning`.", "documented": true, "public": false, "addressable": true, "line": 195, "end_line": 205, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L195-L205", "decorators": [], "calls": [ "make_sampler", "logger.warning" ], "state_reads": [], "state_writes": [ "self._sampler" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "initialize_cache", "qualname": "MLXModelRunner.initialize_cache", "full_name": "vllm_mlx.model_runner.MLXModelRunner.initialize_cache", "kind": "method", "signature": "def initialize_cache(self, num_blocks: int) -> None", "parameters": [ { "name": "num_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Initialize KV cache.", "summary": "Initialize KV cache.", "implementation": "Method `MLXModelRunner.initialize_cache` updates `self._num_cache_blocks`; calls `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 207, "end_line": 210, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L207-L210", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [ "self._num_cache_blocks" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_kv_cache_spec", "qualname": "MLXModelRunner.get_kv_cache_spec", "full_name": "vllm_mlx.model_runner.MLXModelRunner.get_kv_cache_spec", "kind": "method", "signature": "def get_kv_cache_spec(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get KV cache specification.", "summary": "Get KV cache specification.", "implementation": "Method `MLXModelRunner.get_kv_cache_spec` returns `{'num_blocks': self._num_cache_blocks, 'block_size': self.cache_config.block_size}`.", "documented": true, "public": true, "addressable": true, "line": 215, "end_line": 220, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L215-L220", "decorators": [], "calls": [], "state_reads": [ "self._num_cache_blocks", "self.cache_config.block_size", "self.cache_config" ], "state_writes": [], "raises": [], "return_expressions": [ "{'num_blocks': self._num_cache_blocks, 'block_size': self.cache_config.block_size}" ], "awaits": false, "yields": false }, { "name": "get_cache_block_size_bytes", "qualname": "MLXModelRunner.get_cache_block_size_bytes", "full_name": "vllm_mlx.model_runner.MLXModelRunner.get_cache_block_size_bytes", "kind": "method", "signature": "def get_cache_block_size_bytes(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Calculate cache block size in bytes.", "summary": "Calculate cache block size in bytes.", "implementation": "Method `MLXModelRunner.get_cache_block_size_bytes` calls `getattr`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 222, "end_line": 240, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L222-L240", "decorators": [], "calls": [ "getattr" ], "state_reads": [ "self._loaded", "self.model", "self.cache_config.block_size", "self.cache_config" ], "state_writes": [], "raises": [], "return_expressions": [ "0", "2 * block_size * num_layers * num_kv_heads * head_size * 2" ], "awaits": false, "yields": false }, { "name": "warm_up", "qualname": "MLXModelRunner.warm_up", "full_name": "vllm_mlx.model_runner.MLXModelRunner.warm_up", "kind": "method", "signature": "def warm_up(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Warm up model with a test generation.", "summary": "Warm up model with a test generation.", "implementation": "Method `MLXModelRunner.warm_up` calls `self.load_model`, `logger.info`, `generate`, `logger.warning`.", "documented": true, "public": true, "addressable": true, "line": 242, "end_line": 263, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L242-L263", "decorators": [], "calls": [ "self.load_model", "logger.info", "generate", "logger.warning" ], "state_reads": [ "self._loaded", "self.load_model", "self.model", "self.tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "execute_model", "qualname": "MLXModelRunner.execute_model", "full_name": "vllm_mlx.model_runner.MLXModelRunner.execute_model", "kind": "method", "signature": "def execute_model(self, scheduler_output: 'SchedulerOutput') -> MLXModelRunnerOutput", "parameters": [ { "name": "scheduler_output", "kind": "positional or keyword", "annotation": "'SchedulerOutput'", "default": "", "required": true, "description": "Contains requests to process" } ], "return_annotation": "MLXModelRunnerOutput", "docstring": "Execute model inference for scheduled requests.\n\nArgs:\n scheduler_output: Contains requests to process\n\nReturns:\n MLXModelRunnerOutput with generated tokens", "summary": "Execute model inference for scheduled requests.", "implementation": "Method `MLXModelRunner.execute_model` calls `RuntimeError`, `time.time`, `self._generate_for_request`, `len`; can raise `RuntimeError`; returns `MLXModelRunnerOutput(req_id_to_token_ids=req_id_to_token_ids, num_tokens_generated=total_tokens, generation_time_s=gene…`.", "documented": true, "public": true, "addressable": true, "line": 265, "end_line": 315, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L265-L315", "decorators": [], "calls": [ "RuntimeError", "time.time", "self._generate_for_request", "len", "self._continue_generation", "MLXModelRunnerOutput" ], "state_reads": [ "self._loaded", "self._generate_for_request", "self._continue_generation" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "MLXModelRunnerOutput(req_id_to_token_ids=req_id_to_token_ids, num_tokens_generated=total_tokens, generation_time_s=gene…" ], "awaits": false, "yields": false }, { "name": "_prefill_with_chunking", "qualname": "MLXModelRunner._prefill_with_chunking", "full_name": "vllm_mlx.model_runner.MLXModelRunner._prefill_with_chunking", "kind": "method", "signature": "def _prefill_with_chunking(self, input_ids: mx.array, cache: Optional[Any]=None) -> tuple[mx.array, Any]", "parameters": [ { "name": "input_ids", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Input token IDs [1, seq_len]" }, { "name": "cache", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional existing KV cache" } ], "return_annotation": "tuple[mx.array, Any]", "docstring": "Process prompt with optimal chunking for L2 cache efficiency.\n\nLong prompts are broken into chunks that fit in L2 cache,\nmaximizing memory bandwidth utilization during prefill.\n\nArgs:\n input_ids: Input token IDs [1, seq_len]\n cache: Optional existing KV cache\n\nReturns:\n Tuple of (logits, updated_cache)", "summary": "Process prompt with optimal chunking for L2 cache efficiency.", "implementation": "Method `MLXModelRunner._prefill_with_chunking` calls `len`, `get_optimal_prefill_size`, `input_ids.reshape`, `forward_fn`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 317, "end_line": 362, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L317-L362", "decorators": [], "calls": [ "len", "get_optimal_prefill_size", "input_ids.reshape", "forward_fn", "range", "mx.eval" ], "state_reads": [ "self._compiled_forward", "self.model" ], "state_writes": [], "raises": [], "return_expressions": [ "forward_fn(input_ids, cache=cache)", "(logits, cache)" ], "awaits": false, "yields": false }, { "name": "get_optimal_prefill_size", "qualname": "MLXModelRunner._prefill_with_chunking.get_optimal_prefill_size", "full_name": "vllm_mlx.model_runner.MLXModelRunner._prefill_with_chunking.get_optimal_prefill_size", "kind": "nested function", "signature": "def get_optimal_prefill_size(seq_len)", "parameters": [ { "name": "seq_len", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `MLXModelRunner._prefill_with_chunking.get_optimal_prefill_size` calls `min`; returns `min(512, seq_len)`.", "implementation": "Nested Function `MLXModelRunner._prefill_with_chunking.get_optimal_prefill_size` calls `min`; returns `min(512, seq_len)`.", "documented": false, "public": false, "addressable": false, "line": 339, "end_line": 340, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L339-L340", "decorators": [], "calls": [ "min" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "min(512, seq_len)" ], "awaits": false, "yields": false }, { "name": "_generate_for_request", "qualname": "MLXModelRunner._generate_for_request", "full_name": "vllm_mlx.model_runner.MLXModelRunner._generate_for_request", "kind": "method", "signature": "def _generate_for_request(self, prompt_token_ids: list[int], sampling_params: Any, max_tokens: int=1) -> list[int]", "parameters": [ { "name": "prompt_token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Input token IDs" }, { "name": "sampling_params", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Sampling parameters" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Maximum tokens to generate" } ], "return_annotation": "list[int]", "docstring": "Generate tokens for a single request.\n\nUses optimizations when enabled:\n- Compiled forward pass (kernel fusion)\n- Prefill chunking for long prompts\n\nArgs:\n prompt_token_ids: Input token IDs\n sampling_params: Sampling parameters\n max_tokens: Maximum tokens to generate\n\nReturns:\n List of generated token IDs", "summary": "Generate tokens for a single request.", "implementation": "Method `MLXModelRunner._generate_for_request` calls `getattr`, `make_sampler`, `mx.array`, `generate_step`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 364, "end_line": 418, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L364-L418", "decorators": [], "calls": [ "getattr", "make_sampler", "mx.array", "generate_step", "hasattr", "generated_ids.append", "isinstance", "len", "logger.error" ], "state_reads": [ "self.model" ], "state_writes": [], "raises": [], "return_expressions": [ "generated_ids", "[]" ], "awaits": false, "yields": false }, { "name": "_continue_generation", "qualname": "MLXModelRunner._continue_generation", "full_name": "vllm_mlx.model_runner.MLXModelRunner._continue_generation", "kind": "method", "signature": "def _continue_generation(self, req_id: str) -> list[int]", "parameters": [ { "name": "req_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[int]", "docstring": "Continue generation for an existing request.\n\nThis is a placeholder - in a full implementation, we would\nuse cached KV states to continue generation efficiently.", "summary": "Continue generation for an existing request.", "implementation": "Method `MLXModelRunner._continue_generation` returns `[]`.", "documented": true, "public": false, "addressable": true, "line": 420, "end_line": 428, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L420-L428", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]" ], "awaits": false, "yields": false }, { "name": "decode_tokens", "qualname": "MLXModelRunner.decode_tokens", "full_name": "vllm_mlx.model_runner.MLXModelRunner.decode_tokens", "kind": "method", "signature": "def decode_tokens(self, token_ids: list[int]) -> str", "parameters": [ { "name": "token_ids", "kind": "positional or keyword", "annotation": "list[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Decode token IDs to text.", "summary": "Decode token IDs to text.", "implementation": "Method `MLXModelRunner.decode_tokens` calls `self.tokenizer.decode`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 430, "end_line": 434, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L430-L434", "decorators": [], "calls": [ "self.tokenizer.decode" ], "state_reads": [ "self.tokenizer", "self.tokenizer.decode" ], "state_writes": [], "raises": [], "return_expressions": [ "''", "self.tokenizer.decode(token_ids)" ], "awaits": false, "yields": false }, { "name": "get_model_info", "qualname": "MLXModelRunner.get_model_info", "full_name": "vllm_mlx.model_runner.MLXModelRunner.get_model_info", "kind": "method", "signature": "def get_model_info(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get information about the loaded model and optimizations.", "summary": "Get information about the loaded model and optimizations.", "implementation": "Method `MLXModelRunner.get_model_info` calls `getattr`, `info.update`; returns `info`.", "documented": true, "public": true, "addressable": true, "line": 436, "end_line": 471, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L436-L471", "decorators": [], "calls": [ "getattr", "info.update" ], "state_reads": [ "self._loaded", "self.model_config.model", "self.model_config", "self._enable_optimizations", "self.model", "self._compiled_forward", "self._hardware_info", "self._hardware_info.chip_name", "self._hardware_info.total_memory_gb", "self._hardware_info.memory_bandwidth_gbs", "self._hardware_info.gpu_cores", "self._hardware_info.optimal_prefill_size" ], "state_writes": [], "raises": [], "return_expressions": [ "info" ], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "MLXModelRunner.__repr__", "full_name": "vllm_mlx.model_runner.MLXModelRunner.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `MLXModelRunner.__repr__` returns `f''`.", "implementation": "Method `MLXModelRunner.__repr__` returns `f''`.", "documented": false, "public": false, "addressable": true, "line": 473, "end_line": 476, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_runner.py#L473-L476", "decorators": [], "calls": [], "state_reads": [ "self._loaded", "self._compiled_forward", "self.model_config.model", "self.model_config" ], "state_writes": [], "raises": [], "return_expressions": [ "f''" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.model_workflow", "path": "vllm_mlx/model_workflow.py", "page_path": "reference/api/vllm_mlx/model_workflow.md", "docstring": "Model acquisition, inspection, and conversion workflow helpers.\n\nThe functions in this module intentionally avoid loading model weights. They\ncollect repository/file metadata, download artifacts, and record manifests so a\nmodel can be qualified before it is served.", "summary": "Model acquisition, inspection, and conversion workflow helpers.", "line_count": 661, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L1-L661", "members": [ "MODEL_MANIFEST_NAME", "CONVERSION_MANIFEST_NAME", "REGISTRATION_MANIFEST_NAME", "QUALIFICATION_REQUEST_NAME", "_MODEL_ID_RE", "AcquisitionOptions", "ConversionOptions", "RegistrationOptions", "QualificationOptions", "_now_iso", "_bytes_to_gb", "_read_json", "_write_json", "_local_file_inventory", "_hf_file_inventory", "_hf_config", "_config_value", "_model_family", "_estimate_fit", "_model_file_bytes", "_NON_MLX_QUANT_METHODS", "_is_mlx_quantization", "_looks_like_mlx_name", "_is_model_id", "_fast_transfer_env", "inspect_model", "acquire_model", "_conversion_command", "convert_model", "_existing_manifests", "_drop_none", "register_model", "_qualification_command", "qualify_model" ], "symbols": [ { "name": "AcquisitionOptions", "qualname": "AcquisitionOptions", "full_name": "vllm_mlx.model_workflow.AcquisitionOptions", "kind": "class", "signature": "class AcquisitionOptions", "parameters": [ { "name": "revision", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "target_dir", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "staging_dir", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "is_mllm", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "fast_transfer", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "local_files_only", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "AcquisitionOptions", "docstring": "Options for Hugging Face model acquisition.", "summary": "Options for Hugging Face model acquisition.", "implementation": "Class `AcquisitionOptions` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 38, "end_line": 46, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L38-L46", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ConversionOptions", "qualname": "ConversionOptions", "full_name": "vllm_mlx.model_workflow.ConversionOptions", "kind": "class", "signature": "class ConversionOptions", "parameters": [ { "name": "source_path", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "output_path", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "quantize", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "q_bits", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "q_group_size", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "q_mode", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "quant_predicate", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "dtype", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "trust_remote_code", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "dry_run", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "ConversionOptions", "docstring": "Options for the mlx-lm conversion backend.", "summary": "Options for the mlx-lm conversion backend.", "implementation": "Class `ConversionOptions` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 50, "end_line": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L50-L62", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RegistrationOptions", "qualname": "RegistrationOptions", "full_name": "vllm_mlx.model_workflow.RegistrationOptions", "kind": "class", "signature": "class RegistrationOptions", "parameters": [ { "name": "artifact_path", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "model_id", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "served_model_name", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "preset_alias", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "output_path", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "mllm", "kind": "field", "annotation": "bool | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tool_call_parser", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "reasoning_parser", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "default_temperature", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "default_top_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "default_top_k", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "default_min_p", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "default_presence_penalty", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "default_repetition_penalty", "kind": "field", "annotation": "float | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "chat_template_kwargs", "kind": "field", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "feature_flags", "kind": "field", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "RegistrationOptions", "docstring": "Options for generating a portable model registration manifest.", "summary": "Options for generating a portable model registration manifest.", "implementation": "Class `RegistrationOptions` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 66, "end_line": 84, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L66-L84", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "QualificationOptions", "qualname": "QualificationOptions", "full_name": "vllm_mlx.model_workflow.QualificationOptions", "kind": "class", "signature": "class QualificationOptions", "parameters": [ { "name": "model_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "server_url", "kind": "field", "annotation": "str", "default": "'http://127.0.0.1:8080'", "required": false, "description": "Optional constructor field; defaults to `'http://127.0.0.1:8080'`." }, { "name": "workload_path", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "output_path", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "result_path", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "repetitions", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "dry_run", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "extra_args", "kind": "field", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "QualificationOptions", "docstring": "Options for creating or running a bench-serve qualification handoff.", "summary": "Options for creating or running a bench-serve qualification handoff.", "implementation": "Class `QualificationOptions` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 88, "end_line": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L88-L98", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_now_iso", "qualname": "_now_iso", "full_name": "vllm_mlx.model_workflow._now_iso", "kind": "function", "signature": "def _now_iso() -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Function `_now_iso` calls `datetime.now(timezone.utc).isoformat`, `datetime.now`; returns `datetime.now(timezone.utc).isoformat()`.", "implementation": "Function `_now_iso` calls `datetime.now(timezone.utc).isoformat`, `datetime.now`; returns `datetime.now(timezone.utc).isoformat()`.", "documented": false, "public": false, "addressable": true, "line": 101, "end_line": 102, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L101-L102", "decorators": [], "calls": [ "datetime.now(timezone.utc).isoformat", "datetime.now" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "datetime.now(timezone.utc).isoformat()" ], "awaits": false, "yields": false }, { "name": "_bytes_to_gb", "qualname": "_bytes_to_gb", "full_name": "vllm_mlx.model_workflow._bytes_to_gb", "kind": "function", "signature": "def _bytes_to_gb(size: int | float | None) -> float | None", "parameters": [ { "name": "size", "kind": "positional or keyword", "annotation": "int | float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float | None", "docstring": "", "summary": "Function `_bytes_to_gb` calls `round`, `float`; has 2 explicit return paths.", "implementation": "Function `_bytes_to_gb` calls `round`, `float`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 105, "end_line": 108, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L105-L108", "decorators": [], "calls": [ "round", "float" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "round(float(size) / 1024 ** 3, 3)" ], "awaits": false, "yields": false }, { "name": "_read_json", "qualname": "_read_json", "full_name": "vllm_mlx.model_workflow._read_json", "kind": "function", "signature": "def _read_json(path: Path) -> dict[str, Any]", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Function `_read_json` calls `json.loads`, `path.read_text`; has 2 explicit return paths.", "implementation": "Function `_read_json` calls `json.loads`, `path.read_text`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 111, "end_line": 115, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L111-L115", "decorators": [], "calls": [ "json.loads", "path.read_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.loads(path.read_text())", "{}" ], "awaits": false, "yields": false }, { "name": "_write_json", "qualname": "_write_json", "full_name": "vllm_mlx.model_workflow._write_json", "kind": "function", "signature": "def _write_json(path: Path, payload: dict[str, Any]) -> None", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "payload", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Function `_write_json` calls `path.parent.mkdir`, `path.write_text`, `json.dumps`.", "implementation": "Function `_write_json` calls `path.parent.mkdir`, `path.write_text`, `json.dumps`.", "documented": false, "public": false, "addressable": true, "line": 118, "end_line": 120, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L118-L120", "decorators": [], "calls": [ "path.parent.mkdir", "path.write_text", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_local_file_inventory", "qualname": "_local_file_inventory", "full_name": "vllm_mlx.model_workflow._local_file_inventory", "kind": "function", "signature": "def _local_file_inventory(path: Path) -> tuple[list[dict[str, Any]], int]", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[list[dict[str, Any]], int]", "docstring": "", "summary": "Function `_local_file_inventory` calls `sorted`, `path.rglob`, `item.is_file`, `item.stat`; returns `(files, total)`.", "implementation": "Function `_local_file_inventory` calls `sorted`, `path.rglob`, `item.is_file`, `item.stat`; returns `(files, total)`.", "documented": false, "public": false, "addressable": true, "line": 123, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L123-L135", "decorators": [], "calls": [ "sorted", "path.rglob", "item.is_file", "item.stat", "files.append", "str", "item.relative_to" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(files, total)" ], "awaits": false, "yields": false }, { "name": "_hf_file_inventory", "qualname": "_hf_file_inventory", "full_name": "vllm_mlx.model_workflow._hf_file_inventory", "kind": "function", "signature": "def _hf_file_inventory(model_id: str, *, revision: str | None, local_files_only: bool) -> tuple[list[dict[str, Any]], int | None, str | None]", "parameters": [ { "name": "model_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "revision", "kind": "keyword-only", "annotation": "str | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "local_files_only", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "tuple[list[dict[str, Any]], int | None, str | None]", "docstring": "", "summary": "Function `_hf_file_inventory` calls `HfApi().model_info`, `HfApi`, `getattr`, `int`; has 2 explicit return paths.", "implementation": "Function `_hf_file_inventory` calls `HfApi().model_info`, `HfApi`, `getattr`, `int`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 138, "end_line": 158, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L138-L158", "decorators": [], "calls": [ "HfApi().model_info", "HfApi", "getattr", "int", "files.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "([], None, revision)", "(files, total if total_known else None, getattr(info, 'sha', revision))" ], "awaits": false, "yields": false }, { "name": "_hf_config", "qualname": "_hf_config", "full_name": "vllm_mlx.model_workflow._hf_config", "kind": "function", "signature": "def _hf_config(model_id: str, *, revision: str | None, local_files_only: bool) -> dict[str, Any]", "parameters": [ { "name": "model_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "revision", "kind": "keyword-only", "annotation": "str | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "local_files_only", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Function `_hf_config` calls `hf_hub_download`, `_read_json`, `Path`; returns `_read_json(Path(config_path))`.", "implementation": "Function `_hf_config` calls `hf_hub_download`, `_read_json`, `Path`; returns `_read_json(Path(config_path))`.", "documented": false, "public": false, "addressable": true, "line": 161, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L161-L170", "decorators": [], "calls": [ "hf_hub_download", "_read_json", "Path" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_read_json(Path(config_path))" ], "awaits": false, "yields": false }, { "name": "_config_value", "qualname": "_config_value", "full_name": "vllm_mlx.model_workflow._config_value", "kind": "function", "signature": "def _config_value(config: dict[str, Any], key: str) -> Any", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "", "summary": "Function `_config_value` calls `config.get`, `isinstance`, `text_config.get`; has 3 explicit return paths.", "implementation": "Function `_config_value` calls `config.get`, `isinstance`, `text_config.get`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 173, "end_line": 179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L173-L179", "decorators": [], "calls": [ "config.get", "isinstance", "text_config.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "config[key]", "text_config.get(key)", "None" ], "awaits": false, "yields": false }, { "name": "_model_family", "qualname": "_model_family", "full_name": "vllm_mlx.model_workflow._model_family", "kind": "function", "signature": "def _model_family(config: dict[str, Any]) -> dict[str, Any]", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Function `_model_family` calls `_config_value`, `isinstance`, `config.get`; returns `{'model_type': _config_value(config, 'model_type'), 'architectures': architectures, 'torch_dtype': _config_value(config…`.", "implementation": "Function `_model_family` calls `_config_value`, `isinstance`, `config.get`; returns `{'model_type': _config_value(config, 'model_type'), 'architectures': architectures, 'torch_dtype': _config_value(config…`.", "documented": false, "public": false, "addressable": true, "line": 182, "end_line": 201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L182-L201", "decorators": [], "calls": [ "_config_value", "isinstance", "config.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'model_type': _config_value(config, 'model_type'), 'architectures': architectures, 'torch_dtype': _config_value(config…" ], "awaits": false, "yields": false }, { "name": "_estimate_fit", "qualname": "_estimate_fit", "full_name": "vllm_mlx.model_workflow._estimate_fit", "kind": "function", "signature": "def _estimate_fit(*, total_bytes: int | None, model_files_bytes: int | None, config: dict[str, Any]) -> dict[str, Any]", "parameters": [ { "name": "total_bytes", "kind": "keyword-only", "annotation": "int | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "model_files_bytes", "kind": "keyword-only", "annotation": "int | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "config", "kind": "keyword-only", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Function `_estimate_fit` calls `_model_family(config).get`, `_model_family`, `isinstance`, `warnings.append`; returns `{'download_size_gb': _bytes_to_gb(total_bytes), 'model_file_size_gb': _bytes_to_gb(model_files_bytes), 'estimated_conve…`.", "implementation": "Function `_estimate_fit` calls `_model_family(config).get`, `_model_family`, `isinstance`, `warnings.append`; returns `{'download_size_gb': _bytes_to_gb(total_bytes), 'model_file_size_gb': _bytes_to_gb(model_files_bytes), 'estimated_conve…`.", "documented": false, "public": false, "addressable": true, "line": 204, "end_line": 231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L204-L231", "decorators": [], "calls": [ "_model_family(config).get", "_model_family", "isinstance", "warnings.append", "int", "_bytes_to_gb" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'download_size_gb': _bytes_to_gb(total_bytes), 'model_file_size_gb': _bytes_to_gb(model_files_bytes), 'estimated_conve…" ], "awaits": false, "yields": false }, { "name": "_model_file_bytes", "qualname": "_model_file_bytes", "full_name": "vllm_mlx.model_workflow._model_file_bytes", "kind": "function", "signature": "def _model_file_bytes(files: list[dict[str, Any]]) -> int | None", "parameters": [ { "name": "files", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int | None", "docstring": "", "summary": "Function `_model_file_bytes` calls `str`, `entry.get`, `path.endswith`, `int`; has 2 explicit return paths.", "implementation": "Function `_model_file_bytes` calls `str`, `entry.get`, `path.endswith`, `int`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 234, "end_line": 246, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L234-L246", "decorators": [], "calls": [ "str", "entry.get", "path.endswith", "int" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "total if known else None" ], "awaits": false, "yields": false }, { "name": "_is_mlx_quantization", "qualname": "_is_mlx_quantization", "full_name": "vllm_mlx.model_workflow._is_mlx_quantization", "kind": "function", "signature": "def _is_mlx_quantization(quant: Any) -> bool", "parameters": [ { "name": "quant", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True only when *quant* looks like an mlx-lm quantization config.\n\nPyTorch quantization configs (GPTQ, AWQ, ...) carry a ``quant_method``\nkey that MLX configs never set. Treating those as MLX-ready is a false\npositive reported in review.", "summary": "Return True only when *quant* looks like an mlx-lm quantization config.", "implementation": "Function `_is_mlx_quantization` calls `isinstance`, `str(quant.get('quant_method', '')).lower`, `str`, `quant.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 252, "end_line": 265, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L252-L265", "decorators": [], "calls": [ "isinstance", "str(quant.get('quant_method', '')).lower", "str", "quant.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "'bits' in quant" ], "awaits": false, "yields": false }, { "name": "_looks_like_mlx_name", "qualname": "_looks_like_mlx_name", "full_name": "vllm_mlx.model_workflow._looks_like_mlx_name", "kind": "function", "signature": "def _looks_like_mlx_name(model: str, *, source: str) -> bool", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "source", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "bool", "docstring": "", "summary": "Function `_looks_like_mlx_name` calls `model.lower`, `Path(model).name.lower`, `Path`, `name.startswith`; returns `name.startswith('mlx-community/') or '-mlx' in name or '_mlx' in name or name.endswith('mlx')`.", "implementation": "Function `_looks_like_mlx_name` calls `model.lower`, `Path(model).name.lower`, `Path`, `name.startswith`; returns `name.startswith('mlx-community/') or '-mlx' in name or '_mlx' in name or name.endswith('mlx')`.", "documented": false, "public": false, "addressable": true, "line": 268, "end_line": 275, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L268-L275", "decorators": [], "calls": [ "model.lower", "Path(model).name.lower", "Path", "name.startswith", "name.endswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "name.startswith('mlx-community/') or '-mlx' in name or '_mlx' in name or name.endswith('mlx')" ], "awaits": false, "yields": false }, { "name": "_is_model_id", "qualname": "_is_model_id", "full_name": "vllm_mlx.model_workflow._is_model_id", "kind": "function", "signature": "def _is_model_id(value: str) -> bool", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "", "summary": "Function `_is_model_id` calls `bool`, `_MODEL_ID_RE.fullmatch`; returns `bool(_MODEL_ID_RE.fullmatch(value))`.", "implementation": "Function `_is_model_id` calls `bool`, `_MODEL_ID_RE.fullmatch`; returns `bool(_MODEL_ID_RE.fullmatch(value))`.", "documented": false, "public": false, "addressable": true, "line": 278, "end_line": 279, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L278-L279", "decorators": [], "calls": [ "bool", "_MODEL_ID_RE.fullmatch" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "bool(_MODEL_ID_RE.fullmatch(value))" ], "awaits": false, "yields": false }, { "name": "_fast_transfer_env", "qualname": "_fast_transfer_env", "full_name": "vllm_mlx.model_workflow._fast_transfer_env", "kind": "function", "signature": "def _fast_transfer_env(requested: bool) -> tuple[dict[str, str], dict[str, Any]]", "parameters": [ { "name": "requested", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[dict[str, str], dict[str, Any]]", "docstring": "", "summary": "Function `_fast_transfer_env` calls `find_spec`; has 3 explicit return paths.", "implementation": "Function `_fast_transfer_env` calls `find_spec`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 282, "end_line": 297, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L282-L297", "decorators": [], "calls": [ "find_spec" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "({}, {'requested': False, 'enabled': False, 'reason': 'disabled'})", "({}, {'requested': True, 'enabled': False, 'reason': 'hf_transfer package is not installed'})", "({'HF_HUB_ENABLE_HF_TRANSFER': '1'}, {'requested': True, 'enabled': True, 'reason': 'enabled'})" ], "awaits": false, "yields": false }, { "name": "inspect_model", "qualname": "inspect_model", "full_name": "vllm_mlx.model_workflow.inspect_model", "kind": "function", "signature": "def inspect_model(model: str, *, revision: str | None=None, local_files_only: bool=False) -> dict[str, Any]", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "revision", "kind": "keyword-only", "annotation": "str | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "local_files_only", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." } ], "return_annotation": "dict[str, Any]", "docstring": "Inspect a local model path or Hugging Face model id without loading weights.", "summary": "Inspect a local model path or Hugging Face model id without loading weights.", "implementation": "Function `inspect_model` calls `Path(model).expanduser`, `Path`, `model_path.exists`, `_local_file_inventory`; can raise `ValueError`; returns `{'model': model, 'source': source, 'location': location, 'revision': resolved_revision or revision, 'inspected_at': _no…`.", "documented": true, "public": true, "addressable": true, "line": 300, "end_line": 366, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L300-L366", "decorators": [], "calls": [ "Path(model).expanduser", "Path", "model_path.exists", "_local_file_inventory", "_read_json", "str", "_is_model_id", "ValueError", "_hf_file_inventory", "_hf_config", "warnings.append", "_model_file_bytes", "_model_family", "_estimate_fit", "warnings.extend", "estimate.pop", "_looks_like_mlx_name", "bool", "_is_mlx_quantization", "family.get", "_now_iso", "len", "_bytes_to_gb" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "{'model': model, 'source': source, 'location': location, 'revision': resolved_revision or revision, 'inspected_at': _no…" ], "awaits": false, "yields": false }, { "name": "acquire_model", "qualname": "acquire_model", "full_name": "vllm_mlx.model_workflow.acquire_model", "kind": "function", "signature": "def acquire_model(model_id: str, *, options: AcquisitionOptions | None=None) -> dict[str, Any]", "parameters": [ { "name": "model_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "options", "kind": "keyword-only", "annotation": "AcquisitionOptions | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "dict[str, Any]", "docstring": "Download a model repository and write a finalized artifact manifest.", "summary": "Download a model repository and write a finalized artifact manifest.", "implementation": "Function `acquire_model` calls `AcquisitionOptions`, `_is_model_id`, `ValueError`, `_fast_transfer_env`; can raise `ValueError`, `FileExistsError`; returns `manifest`.", "documented": true, "public": true, "addressable": true, "line": 369, "end_line": 446, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L369-L446", "decorators": [], "calls": [ "AcquisitionOptions", "_is_model_id", "ValueError", "_fast_transfer_env", "os.environ.get", "os.environ.update", "Path(options.target_dir).expanduser", "Path", "target.exists", "FileExistsError", "Path(options.staging_dir).expanduser", "staging_root.mkdir", "tempfile.mkdtemp", "snapshot_download", "str", "target.parent.mkdir", "shutil.move", "shutil.rmtree", "old_env.items", "os.environ.pop", "inspect_model", "_now_iso", "_write_json" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError", "FileExistsError" ], "return_expressions": [ "manifest" ], "awaits": false, "yields": false }, { "name": "_conversion_command", "qualname": "_conversion_command", "full_name": "vllm_mlx.model_workflow._conversion_command", "kind": "function", "signature": "def _conversion_command(options: ConversionOptions) -> list[str]", "parameters": [ { "name": "options", "kind": "positional or keyword", "annotation": "ConversionOptions", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "", "summary": "Function `_conversion_command` calls `command.append`, `command.extend`, `str`; returns `command`.", "implementation": "Function `_conversion_command` calls `command.append`, `command.extend`, `str`; returns `command`.", "documented": false, "public": false, "addressable": true, "line": 449, "end_line": 474, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L449-L474", "decorators": [], "calls": [ "command.append", "command.extend", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "command" ], "awaits": false, "yields": false }, { "name": "convert_model", "qualname": "convert_model", "full_name": "vllm_mlx.model_workflow.convert_model", "kind": "function", "signature": "def convert_model(options: ConversionOptions) -> dict[str, Any]", "parameters": [ { "name": "options", "kind": "positional or keyword", "annotation": "ConversionOptions", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Run mlx-lm conversion and record the exact recipe.", "summary": "Run mlx-lm conversion and record the exact recipe.", "implementation": "Function `convert_model` calls `_conversion_command`, `_now_iso`, `inspect_model`, `sys.version.split`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 477, "end_line": 525, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L477-L525", "decorators": [], "calls": [ "_conversion_command", "_now_iso", "inspect_model", "sys.version.split", "platform.platform", "subprocess.run", "Path(options.output_path).expanduser", "Path", "str", "_write_json" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "_existing_manifests", "qualname": "_existing_manifests", "full_name": "vllm_mlx.model_workflow._existing_manifests", "kind": "function", "signature": "def _existing_manifests(path: Path) -> dict[str, Any]", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Function `_existing_manifests` calls `manifest_path.exists`, `str`, `_read_json`; returns `manifests`.", "implementation": "Function `_existing_manifests` calls `manifest_path.exists`, `str`, `_read_json`; returns `manifests`.", "documented": false, "public": false, "addressable": true, "line": 528, "end_line": 540, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L528-L540", "decorators": [], "calls": [ "manifest_path.exists", "str", "_read_json" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "manifests" ], "awaits": false, "yields": false }, { "name": "_drop_none", "qualname": "_drop_none", "full_name": "vllm_mlx.model_workflow._drop_none", "kind": "function", "signature": "def _drop_none(payload: dict[str, Any]) -> dict[str, Any]", "parameters": [ { "name": "payload", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "", "summary": "Function `_drop_none` calls `payload.items`; returns `{key: value for key, value in payload.items() if value is not None}`.", "implementation": "Function `_drop_none` calls `payload.items`; returns `{key: value for key, value in payload.items() if value is not None}`.", "documented": false, "public": false, "addressable": true, "line": 543, "end_line": 544, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L543-L544", "decorators": [], "calls": [ "payload.items" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{key: value for key, value in payload.items() if value is not None}" ], "awaits": false, "yields": false }, { "name": "register_model", "qualname": "register_model", "full_name": "vllm_mlx.model_workflow.register_model", "kind": "function", "signature": "def register_model(options: RegistrationOptions) -> dict[str, Any]", "parameters": [ { "name": "options", "kind": "positional or keyword", "annotation": "RegistrationOptions", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Write a portable registration manifest for a finalized local artifact.\n\nThis deliberately does not mutate a production registry. The manifest is a\nhandoff artifact that Ops or a deployment tool can apply after qualification.", "summary": "Write a portable registration manifest for a finalized local artifact.", "implementation": "Function `register_model` calls `Path(options.artifact_path).expanduser`, `Path`, `artifact.exists`, `FileNotFoundError`; can raise `FileNotFoundError`, `NotADirectoryError`; returns `payload`.", "documented": true, "public": true, "addressable": true, "line": 547, "end_line": 603, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L547-L603", "decorators": [], "calls": [ "Path(options.artifact_path).expanduser", "Path", "artifact.exists", "FileNotFoundError", "artifact.is_dir", "NotADirectoryError", "inspect_model", "str", "_drop_none", "_now_iso", "_existing_manifests", "Path(options.output_path).expanduser", "_write_json" ], "state_reads": [], "state_writes": [], "raises": [ "FileNotFoundError", "NotADirectoryError" ], "return_expressions": [ "payload" ], "awaits": false, "yields": false }, { "name": "_qualification_command", "qualname": "_qualification_command", "full_name": "vllm_mlx.model_workflow._qualification_command", "kind": "function", "signature": "def _qualification_command(options: QualificationOptions) -> list[str]", "parameters": [ { "name": "options", "kind": "positional or keyword", "annotation": "QualificationOptions", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "", "summary": "Function `_qualification_command` calls `command.extend`, `str`; returns `command`.", "implementation": "Function `_qualification_command` calls `command.extend`, `str`; returns `command`.", "documented": false, "public": false, "addressable": true, "line": 606, "end_line": 627, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L606-L627", "decorators": [], "calls": [ "command.extend", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "command" ], "awaits": false, "yields": false }, { "name": "qualify_model", "qualname": "qualify_model", "full_name": "vllm_mlx.model_workflow.qualify_model", "kind": "function", "signature": "def qualify_model(options: QualificationOptions) -> dict[str, Any]", "parameters": [ { "name": "options", "kind": "positional or keyword", "annotation": "QualificationOptions", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Create or run a bench-serve qualification handoff.", "summary": "Create or run a bench-serve qualification handoff.", "implementation": "Function `qualify_model` calls `_qualification_command`, `_now_iso`, `subprocess.run`, `Path(options.output_path).expanduser`; returns `payload`.", "documented": true, "public": true, "addressable": true, "line": 630, "end_line": 661, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/model_workflow.py#L630-L661", "decorators": [], "calls": [ "_qualification_command", "_now_iso", "subprocess.run", "Path(options.output_path).expanduser", "Path", "_write_json", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "payload" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.models", "path": "vllm_mlx/models/__init__.py", "page_path": "reference/api/vllm_mlx/models/index.md", "docstring": "MLX Model wrappers for vLLM.\n\nThis module provides wrappers around mlx-lm and mlx-vlm for\nintegration with vLLM's model execution system.", "summary": "MLX Model wrappers for vLLM.", "line_count": 15, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/__init__.py#L1-L15", "members": [ "MLXVisionLanguageModel", "__all__" ], "symbols": [] }, { "name": "vllm_mlx.models.llm", "path": "vllm_mlx/models/llm.py", "page_path": "reference/api/vllm_mlx/models/llm.md", "docstring": "MLX Language Model wrapper.\n\nThis module provides a wrapper around mlx-lm for LLM inference,\nintegrating with vLLM's model execution system.", "summary": "MLX Language Model wrapper.", "line_count": 422, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L1-L422", "members": [ "logger", "GenerationOutput", "StreamingOutput", "MLXLanguageModel" ], "symbols": [ { "name": "GenerationOutput", "qualname": "GenerationOutput", "full_name": "vllm_mlx.models.llm.GenerationOutput", "kind": "class", "signature": "class GenerationOutput", "parameters": [ { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens", "kind": "field", "annotation": "list[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "finish_reason", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "GenerationOutput", "docstring": "Output from text generation.", "summary": "Output from text generation.", "implementation": "Class `GenerationOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 21, "end_line": 26, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L21-L26", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "StreamingOutput", "qualname": "StreamingOutput", "full_name": "vllm_mlx.models.llm.StreamingOutput", "kind": "class", "signature": "class StreamingOutput", "parameters": [ { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "token", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "finished", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "finish_reason", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "StreamingOutput", "docstring": "Streaming output chunk.", "summary": "Streaming output chunk.", "implementation": "Class `StreamingOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 30, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L30-L37", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLXLanguageModel", "qualname": "MLXLanguageModel", "full_name": "vllm_mlx.models.llm.MLXLanguageModel", "kind": "class", "signature": "class MLXLanguageModel", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "tokenizer_name", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional separate tokenizer name" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "mtp", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable native MTP speculative decoding (model must have MTP head)" }, { "name": "mtp_num_draft_tokens", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Draft tokens per speculative MTP step" } ], "return_annotation": "MLXLanguageModel", "docstring": "Wrapper around mlx-lm for LLM inference.\n\nThis class provides a unified interface for loading and running\ninference on language models using Apple's MLX framework.\n\nExample:\n >>> model = MLXLanguageModel(\"mlx-community/Llama-3.2-3B-Instruct-4bit\")\n >>> output = model.generate(\"Hello, how are you?\", max_tokens=100)\n >>> print(output.text)", "summary": "Wrapper around mlx-lm for LLM inference.", "implementation": "Class `MLXLanguageModel` declares 9 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 422, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L40-L422", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLXLanguageModel.__init__", "full_name": "vllm_mlx.models.llm.MLXLanguageModel.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, tokenizer_name: str | None=None, trust_remote_code: bool=False, mtp: bool=False, mtp_num_draft_tokens: int=1)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "tokenizer_name", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional separate tokenizer name" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "mtp", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable native MTP speculative decoding (model must have MTP head)" }, { "name": "mtp_num_draft_tokens", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Draft tokens per speculative MTP step" } ], "return_annotation": "not annotated", "docstring": "Initialize the MLX language model.\n\nArgs:\n model_name: HuggingFace model name or local path\n tokenizer_name: Optional separate tokenizer name\n trust_remote_code: Whether to trust remote code\n mtp: Enable native MTP speculative decoding (model must have MTP head)\n mtp_num_draft_tokens: Draft tokens per speculative MTP step", "summary": "Initialize the MLX language model.", "implementation": "Method `MLXLanguageModel.__init__` updates `self.model_name`, `self.tokenizer_name`, `self.trust_remote_code`, `self._mtp`.", "documented": true, "public": false, "addressable": true, "line": 53, "end_line": 79, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L53-L79", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.model_name", "self.tokenizer_name", "self.trust_remote_code", "self._mtp", "self._mtp_num_draft_tokens", "self.model", "self.tokenizer", "self._loaded" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load", "qualname": "MLXLanguageModel.load", "full_name": "vllm_mlx.models.llm.MLXLanguageModel.load", "kind": "method", "signature": "def load(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the model and tokenizer.", "summary": "Load the model and tokenizer.", "implementation": "Method `MLXLanguageModel.load` updates `self.model`, `self.tokenizer`, `self._loaded`; calls `logger.info`, `self.model_name.lower`, `load_model_with_fallback`, `ImportError`; can raise `ImportError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 81, "end_line": 114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L81-L114", "decorators": [], "calls": [ "logger.info", "self.model_name.lower", "load_model_with_fallback", "ImportError", "logger.error" ], "state_reads": [ "self._loaded", "self.model_name", "self.trust_remote_code", "self.model_name.lower" ], "state_writes": [ "self.model", "self.tokenizer", "self._loaded" ], "raises": [ "ImportError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_create_sampler", "qualname": "MLXLanguageModel._create_sampler", "full_name": "vllm_mlx.models.llm.MLXLanguageModel._create_sampler", "kind": "method", "signature": "def _create_sampler(self, temperature: float=0.7, top_p: float=0.9, top_k: int=0, min_p: float=0.0)", "parameters": [ { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Optional positional or keyword input; defaults to `0.7`." }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Optional positional or keyword input; defaults to `0.9`." }, { "name": "top_k", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." }, { "name": "min_p", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Optional positional or keyword input; defaults to `0.0`." } ], "return_annotation": "not annotated", "docstring": "Create a sampler for text generation.", "summary": "Create a sampler for text generation.", "implementation": "Method `MLXLanguageModel._create_sampler` calls `make_sampler`; returns `make_sampler(temp=temperature, top_p=top_p, top_k=top_k, min_p=min_p)`.", "documented": true, "public": false, "addressable": true, "line": 116, "end_line": 131, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L116-L131", "decorators": [], "calls": [ "make_sampler" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "make_sampler(temp=temperature, top_p=top_p, top_k=top_k, min_p=min_p)" ], "awaits": false, "yields": false }, { "name": "_create_logits_processors", "qualname": "MLXLanguageModel._create_logits_processors", "full_name": "vllm_mlx.models.llm.MLXLanguageModel._create_logits_processors", "kind": "method", "signature": "def _create_logits_processors(self, presence_penalty: float=0.0, repetition_penalty: float=1.0)", "parameters": [ { "name": "presence_penalty", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Optional positional or keyword input; defaults to `0.0`." }, { "name": "repetition_penalty", "kind": "positional or keyword", "annotation": "float", "default": "1.0", "required": false, "description": "Optional positional or keyword input; defaults to `1.0`." } ], "return_annotation": "not annotated", "docstring": "Create logits processors for penalty-based sampling.", "summary": "Create logits processors for penalty-based sampling.", "implementation": "Method `MLXLanguageModel._create_logits_processors` calls `make_logits_processors`; returns `processors if processors else None`.", "documented": true, "public": false, "addressable": true, "line": 133, "end_line": 147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L133-L147", "decorators": [], "calls": [ "make_logits_processors" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "processors if processors else None" ], "awaits": false, "yields": false }, { "name": "generate", "qualname": "MLXLanguageModel.generate", "full_name": "vllm_mlx.models.llm.MLXLanguageModel.generate", "kind": "method", "signature": "def generate(self, prompt: str, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, top_k: int=0, min_p: float=0.0, presence_penalty: float=0.0, repetition_penalty: float=1.0, stop: list[str] | None=None, logits_processors: list | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Input prompt text" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum number of tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature (0 = greedy)" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p (nucleus) sampling parameter" }, { "name": "top_k", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Top-k sampling (0 = disabled)" }, { "name": "min_p", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Minimum probability threshold" }, { "name": "presence_penalty", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Additive penalty for token presence" }, { "name": "repetition_penalty", "kind": "positional or keyword", "annotation": "float", "default": "1.0", "required": false, "description": "Multiplicative penalty for repeating tokens" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "List of stop sequences" }, { "name": "logits_processors", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "Optional externally-supplied logits processors (e.g. JSON schema constrained decoding). Merged with built-in penalty processors." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "GenerationOutput", "docstring": "Generate text from a prompt.\n\nArgs:\n prompt: Input prompt text\n max_tokens: Maximum number of tokens to generate\n temperature: Sampling temperature (0 = greedy)\n top_p: Top-p (nucleus) sampling parameter\n top_k: Top-k sampling (0 = disabled)\n min_p: Minimum probability threshold\n presence_penalty: Additive penalty for token presence\n repetition_penalty: Multiplicative penalty for repeating tokens\n stop: List of stop sequences\n logits_processors: Optional externally-supplied logits processors\n (e.g. JSON schema constrained decoding). Merged with built-in\n penalty processors.\n\nReturns:\n GenerationOutput with generated text and tokens", "summary": "Generate text from a prompt.", "implementation": "Method `MLXLanguageModel.generate` calls `self.load`, `self._create_sampler`, `self._create_logits_processors`, `list`; returns `GenerationOutput(text=output_text, tokens=tokens, finish_reason=finish_reason)`.", "documented": true, "public": true, "addressable": true, "line": 149, "end_line": 219, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L149-L219", "decorators": [], "calls": [ "self.load", "self._create_sampler", "self._create_logits_processors", "list", "generate", "self.tokenizer.encode", "len", "GenerationOutput" ], "state_reads": [ "self._loaded", "self.load", "self._create_sampler", "self._create_logits_processors", "self.model", "self.tokenizer", "self.tokenizer.encode" ], "state_writes": [], "raises": [], "return_expressions": [ "GenerationOutput(text=output_text, tokens=tokens, finish_reason=finish_reason)" ], "awaits": false, "yields": false }, { "name": "stream_generate", "qualname": "MLXLanguageModel.stream_generate", "full_name": "vllm_mlx.models.llm.MLXLanguageModel.stream_generate", "kind": "method", "signature": "def stream_generate(self, prompt: Union[str, 'mx.array', list[int]], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, top_k: int=0, min_p: float=0.0, presence_penalty: float=0.0, repetition_penalty: float=1.0, stop: list[str] | None=None, logits_processors: list | None=None, prompt_cache=None, **kwargs) -> Iterator[StreamingOutput]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "Union[str, 'mx.array', list[int]]", "default": "", "required": true, "description": "Input prompt text, token array, or token id list" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum number of tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature (0 = greedy)" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p (nucleus) sampling parameter" }, { "name": "top_k", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Top-k sampling (0 = disabled)" }, { "name": "min_p", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Minimum probability threshold" }, { "name": "presence_penalty", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Additive penalty for token presence" }, { "name": "repetition_penalty", "kind": "positional or keyword", "annotation": "float", "default": "1.0", "required": false, "description": "Multiplicative penalty for repeating tokens" }, { "name": "stop", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "List of stop sequences" }, { "name": "logits_processors", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Pre-populated KV cache (e.g. from SpecPrefill)" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "Iterator[StreamingOutput]", "docstring": "Stream text generation token by token.\n\nArgs:\n prompt: Input prompt text, token array, or token id list\n max_tokens: Maximum number of tokens to generate\n temperature: Sampling temperature (0 = greedy)\n top_p: Top-p (nucleus) sampling parameter\n top_k: Top-k sampling (0 = disabled)\n min_p: Minimum probability threshold\n presence_penalty: Additive penalty for token presence\n repetition_penalty: Multiplicative penalty for repeating tokens\n stop: List of stop sequences\n prompt_cache: Pre-populated KV cache (e.g. from SpecPrefill)\n\nYields:\n StreamingOutput for each generated token", "summary": "Stream text generation token by token.", "implementation": "Method `MLXLanguageModel.stream_generate` calls `self.load`, `self._create_sampler`, `self._create_logits_processors`, `isinstance`; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 221, "end_line": 325, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L221-L325", "decorators": [], "calls": [ "self.load", "self._create_sampler", "self._create_logits_processors", "isinstance", "len", "self.tokenizer.encode", "max", "enumerate", "stream_generate", "StreamingOutput", "hasattr" ], "state_reads": [ "self._loaded", "self.load", "self._create_sampler", "self._create_logits_processors", "self.tokenizer.encode", "self.tokenizer", "self._mtp", "self._mtp_num_draft_tokens", "self.model" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "chat", "qualname": "MLXLanguageModel.chat", "full_name": "vllm_mlx.models.llm.MLXLanguageModel.chat", "kind": "method", "signature": "def chat(self, messages: list[dict], max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, tools: list | None=None, chat_template_kwargs: dict | None=None, **kwargs) -> GenerationOutput", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "List of chat messages [{\"role\": \"user\", \"content\": \"...\"}]" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling parameter" }, { "name": "tools", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "Optional list of tools for function calling" }, { "name": "chat_template_kwargs", "kind": "positional or keyword", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional generation parameters" } ], "return_annotation": "GenerationOutput", "docstring": "Generate a chat response.\n\nArgs:\n messages: List of chat messages [{\"role\": \"user\", \"content\": \"...\"}]\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling parameter\n tools: Optional list of tools for function calling\n **kwargs: Additional generation parameters\n\nReturns:\n GenerationOutput with the assistant's response", "summary": "Generate a chat response.", "implementation": "Method `MLXLanguageModel.chat` calls `self.load`, `hasattr`, `template_kwargs.update`, `self.tokenizer.apply_chat_template`; returns `self.generate(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, **kwargs)`.", "documented": true, "public": true, "addressable": true, "line": 327, "end_line": 393, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L327-L393", "decorators": [], "calls": [ "self.load", "hasattr", "template_kwargs.update", "self.tokenizer.apply_chat_template", "template_kwargs.pop", "(chat_template_kwargs or {}).keys", "'\\n'.join", "self.generate" ], "state_reads": [ "self._loaded", "self.load", "self.tokenizer", "self.tokenizer.apply_chat_template", "self.generate" ], "state_writes": [], "raises": [], "return_expressions": [ "self.generate(prompt=prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p, **kwargs)" ], "awaits": false, "yields": false }, { "name": "get_model_info", "qualname": "MLXLanguageModel.get_model_info", "full_name": "vllm_mlx.models.llm.MLXLanguageModel.get_model_info", "kind": "method", "signature": "def get_model_info(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get information about the loaded model.", "summary": "Get information about the loaded model.", "implementation": "Method `MLXLanguageModel.get_model_info` calls `hasattr`, `info.update`, `getattr`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 395, "end_line": 418, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L395-L418", "decorators": [], "calls": [ "hasattr", "info.update", "getattr" ], "state_reads": [ "self._loaded", "self.model_name", "self.tokenizer_name", "self.model", "self.model.config" ], "state_writes": [], "raises": [], "return_expressions": [ "{'loaded': False, 'model_name': self.model_name}", "info" ], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "MLXLanguageModel.__repr__", "full_name": "vllm_mlx.models.llm.MLXLanguageModel.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `MLXLanguageModel.__repr__` returns `f''`.", "implementation": "Method `MLXLanguageModel.__repr__` returns `f''`.", "documented": false, "public": false, "addressable": true, "line": 420, "end_line": 422, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/llm.py#L420-L422", "decorators": [], "calls": [], "state_reads": [ "self._loaded", "self.model_name" ], "state_writes": [], "raises": [], "return_expressions": [ "f''" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.models.mllm", "path": "vllm_mlx/models/mllm.py", "page_path": "reference/api/vllm_mlx/models/mllm.md", "docstring": "MLX Multimodal Language Model (MLLM) wrapper.\n\nThis module provides a wrapper around mlx-vlm for multimodal inference,\nsupporting vision, audio, and video understanding on Apple Silicon.\n\nFeatures:\n- OpenAI-compatible API format for images and video\n- Smart video frame extraction with configurable FPS\n- Base64 and URL image support\n- Streaming generation\n- MLLM KV cache for repeated image/video+prompt combinations", "summary": "MLX Multimodal Language Model (MLLM) wrapper.", "line_count": 2944, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1-L2944", "members": [ "logger", "TempFileManager", "_temp_manager", "cleanup_temp_file", "cleanup_all_temp_files", "FRAME_FACTOR", "DEFAULT_FPS", "MIN_FRAMES", "MAX_FRAMES", "IMAGE_FACTOR", "MAX_IMAGE_SIZE", "MAX_VIDEO_SIZE", "MAX_AUDIO_SIZE", "MAX_BASE64_IMAGE_LENGTH", "MAX_BASE64_VIDEO_LENGTH", "MAX_BASE64_AUDIO_LENGTH", "FileSizeExceededError", "UnsafeRemoteURLError", "_normalize_content_part", "_extract_media_url", "_text_content_part", "_append_text_content_part", "_build_string_mllm_message_content", "_append_ordered_mllm_content_part", "_build_ordered_mllm_message_content", "_normalize_mllm_tool_calls", "_build_mllm_chat_messages", "MultimodalInput", "MLLMOutput", "load_gemma4_assistant_drafter", "_DRAFT_KWARG_NAMES", "_count_draft_tokens", "_install_draft_metrics_hooks", "is_base64_image", "is_url", "is_base64_video", "is_base64_audio", "decode_base64_image", "_validate_url_safety", "_request_with_safe_redirects", "download_image", "_VIDEO_EXT_MAP", "_AUDIO_EXT_MAP", "_download_media", "download_video", "download_audio", "decode_base64_video", "decode_base64_audio", "process_video_input", "process_audio_input", "_video_has_audio_track", "_model_has_sound_encoder", "extract_audio_from_video", "_base64_image_cache", "save_base64_image", "process_image_input", "round_by_factor", "ceil_by_factor", "floor_by_factor", "smart_nframes", "extract_video_frames_smart", "save_frames_to_temp", "MLXMultimodalLM", "MLXVisionLanguageModel", "VLMOutput", "is_vlm_model" ], "symbols": [ { "name": "TempFileManager", "qualname": "TempFileManager", "full_name": "vllm_mlx.models.mllm.TempFileManager", "kind": "class", "signature": "class TempFileManager", "parameters": [], "return_annotation": "TempFileManager", "docstring": "Thread-safe manager for tracking and cleaning up temporary files.", "summary": "Thread-safe manager for tracking and cleaning up temporary files.", "implementation": "Class `TempFileManager` declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 41, "end_line": 86, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L41-L86", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "TempFileManager.__init__", "full_name": "vllm_mlx.models.mllm.TempFileManager.__init__", "kind": "method", "signature": "def __init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `TempFileManager.__init__` updates `self._files`, `self._lock`; calls `set`, `threading.Lock`, `atexit.register`.", "implementation": "Method `TempFileManager.__init__` updates `self._files`, `self._lock`; calls `set`, `threading.Lock`, `atexit.register`.", "documented": false, "public": false, "addressable": true, "line": 44, "end_line": 47, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L44-L47", "decorators": [], "calls": [ "set", "threading.Lock", "atexit.register" ], "state_reads": [ "self.cleanup_all" ], "state_writes": [ "self._files", "self._lock" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "register", "qualname": "TempFileManager.register", "full_name": "vllm_mlx.models.mllm.TempFileManager.register", "kind": "method", "signature": "def register(self, path: str) -> str", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Register a temp file for tracking. Returns the path for convenience.", "summary": "Register a temp file for tracking.", "implementation": "Method `TempFileManager.register` calls `self._files.add`; returns `path`.", "documented": true, "public": true, "addressable": true, "line": 49, "end_line": 53, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L49-L53", "decorators": [], "calls": [ "self._files.add" ], "state_reads": [ "self._lock", "self._files.add", "self._files" ], "state_writes": [], "raises": [], "return_expressions": [ "path" ], "awaits": false, "yields": false }, { "name": "cleanup", "qualname": "TempFileManager.cleanup", "full_name": "vllm_mlx.models.mllm.TempFileManager.cleanup", "kind": "method", "signature": "def cleanup(self, path: str) -> bool", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Clean up a specific temp file. Returns True if successful.", "summary": "Clean up a specific temp file.", "implementation": "Method `TempFileManager.cleanup` calls `self._files.discard`, `os.path.exists`, `os.unlink`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 55, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L55-L67", "decorators": [], "calls": [ "self._files.discard", "os.path.exists", "os.unlink", "logger.debug", "logger.warning" ], "state_reads": [ "self._lock", "self._files", "self._files.discard" ], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "cleanup_all", "qualname": "TempFileManager.cleanup_all", "full_name": "vllm_mlx.models.mllm.TempFileManager.cleanup_all", "kind": "method", "signature": "def cleanup_all(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Clean up all tracked temp files. Returns count of cleaned files.", "summary": "Clean up all tracked temp files.", "implementation": "Method `TempFileManager.cleanup_all` calls `list`, `self._files.clear`, `os.path.exists`, `os.unlink`; returns `cleaned`.", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 86, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L69-L86", "decorators": [], "calls": [ "list", "self._files.clear", "os.path.exists", "os.unlink", "logger.info" ], "state_reads": [ "self._lock", "self._files", "self._files.clear" ], "state_writes": [], "raises": [], "return_expressions": [ "cleaned" ], "awaits": false, "yields": false }, { "name": "cleanup_temp_file", "qualname": "cleanup_temp_file", "full_name": "vllm_mlx.models.mllm.cleanup_temp_file", "kind": "function", "signature": "def cleanup_temp_file(path: str) -> bool", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Clean up a specific temporary file.", "summary": "Clean up a specific temporary file.", "implementation": "Function `cleanup_temp_file` calls `_temp_manager.cleanup`; returns `_temp_manager.cleanup(path)`.", "documented": true, "public": true, "addressable": true, "line": 93, "end_line": 95, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L93-L95", "decorators": [], "calls": [ "_temp_manager.cleanup" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_temp_manager.cleanup(path)" ], "awaits": false, "yields": false }, { "name": "cleanup_all_temp_files", "qualname": "cleanup_all_temp_files", "full_name": "vllm_mlx.models.mllm.cleanup_all_temp_files", "kind": "function", "signature": "def cleanup_all_temp_files() -> int", "parameters": [], "return_annotation": "int", "docstring": "Clean up all tracked temporary files. Returns count of cleaned files.", "summary": "Clean up all tracked temporary files.", "implementation": "Function `cleanup_all_temp_files` calls `_temp_manager.cleanup_all`; returns `_temp_manager.cleanup_all()`.", "documented": true, "public": true, "addressable": true, "line": 98, "end_line": 100, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L98-L100", "decorators": [], "calls": [ "_temp_manager.cleanup_all" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_temp_manager.cleanup_all()" ], "awaits": false, "yields": false }, { "name": "FileSizeExceededError", "qualname": "FileSizeExceededError", "full_name": "vllm_mlx.models.mllm.FileSizeExceededError", "kind": "class", "signature": "class FileSizeExceededError(Exception)", "parameters": [], "return_annotation": "FileSizeExceededError", "docstring": "Raised when a downloaded file exceeds the size limit.", "summary": "Raised when a downloaded file exceeds the size limit.", "implementation": "Class `FileSizeExceededError` derives from `Exception` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 119, "end_line": 122, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L119-L122", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "UnsafeRemoteURLError", "qualname": "UnsafeRemoteURLError", "full_name": "vllm_mlx.models.mllm.UnsafeRemoteURLError", "kind": "class", "signature": "class UnsafeRemoteURLError(ValueError)", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "public_message", "kind": "keyword-only", "annotation": "str", "default": "'Remote media URL is not allowed'", "required": false, "description": "Optional keyword-only input; defaults to `'Remote media URL is not allowed'`." } ], "return_annotation": "UnsafeRemoteURLError", "docstring": "Raised when a remote media URL targets an unsafe destination.", "summary": "Raised when a remote media URL targets an unsafe destination.", "implementation": "Class `UnsafeRemoteURLError` derives from `ValueError` and declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 125, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L125-L135", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "UnsafeRemoteURLError.__init__", "full_name": "vllm_mlx.models.mllm.UnsafeRemoteURLError.__init__", "kind": "method", "signature": "def __init__(self, message: str, *, public_message: str='Remote media URL is not allowed') -> None", "parameters": [ { "name": "message", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "public_message", "kind": "keyword-only", "annotation": "str", "default": "'Remote media URL is not allowed'", "required": false, "description": "Optional keyword-only input; defaults to `'Remote media URL is not allowed'`." } ], "return_annotation": "None", "docstring": "", "summary": "Method `UnsafeRemoteURLError.__init__` updates `self.public_message`; calls `super().__init__`, `super`.", "implementation": "Method `UnsafeRemoteURLError.__init__` updates `self.public_message`; calls `super().__init__`, `super`.", "documented": false, "public": false, "addressable": true, "line": 128, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L128-L135", "decorators": [], "calls": [ "super().__init__", "super" ], "state_reads": [], "state_writes": [ "self.public_message" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_normalize_content_part", "qualname": "_normalize_content_part", "full_name": "vllm_mlx.models.mllm._normalize_content_part", "kind": "function", "signature": "def _normalize_content_part(item: object) -> object", "parameters": [ { "name": "item", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "object", "docstring": "Convert Pydantic content parts into plain Python objects.", "summary": "Convert Pydantic content parts into plain Python objects.", "implementation": "Function `_normalize_content_part` calls `hasattr`, `item.model_dump`, `item.dict().items`, `item.dict`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 138, "end_line": 144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L138-L144", "decorators": [], "calls": [ "hasattr", "item.model_dump", "item.dict().items", "item.dict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "item.model_dump(exclude_none=True)", "{k: v for k, v in item.dict().items() if v is not None}", "item" ], "awaits": false, "yields": false }, { "name": "_extract_media_url", "qualname": "_extract_media_url", "full_name": "vllm_mlx.models.mllm._extract_media_url", "kind": "function", "signature": "def _extract_media_url(item: dict, item_type: str) -> str", "parameters": [ { "name": "item", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "item_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Function `_extract_media_url` calls `item.get`, `isinstance`, `media_value.get`; has 2 explicit return paths.", "implementation": "Function `_extract_media_url` calls `item.get`, `isinstance`, `media_value.get`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 147, "end_line": 161, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L147-L161", "decorators": [], "calls": [ "item.get", "isinstance", "media_value.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''", "media_value if isinstance(media_value, str) else ''" ], "awaits": false, "yields": false }, { "name": "_text_content_part", "qualname": "_text_content_part", "full_name": "vllm_mlx.models.mllm._text_content_part", "kind": "function", "signature": "def _text_content_part(text: str) -> dict[str, str]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, str]", "docstring": "", "summary": "Function `_text_content_part` returns `{'type': 'text', 'text': text, 'content': text}`.", "implementation": "Function `_text_content_part` returns `{'type': 'text', 'text': text, 'content': text}`.", "documented": false, "public": false, "addressable": true, "line": 164, "end_line": 165, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L164-L165", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'type': 'text', 'text': text, 'content': text}" ], "awaits": false, "yields": false }, { "name": "_append_text_content_part", "qualname": "_append_text_content_part", "full_name": "vllm_mlx.models.mllm._append_text_content_part", "kind": "function", "signature": "def _append_text_content_part(built_parts: list[dict[str, str]], text_parts: list[str], text: str) -> None", "parameters": [ { "name": "built_parts", "kind": "positional or keyword", "annotation": "list[dict[str, str]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "text_parts", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Function `_append_text_content_part` calls `built_parts.append`, `_text_content_part`, `text_parts.append`; returns `None`.", "implementation": "Function `_append_text_content_part` calls `built_parts.append`, `_text_content_part`, `text_parts.append`; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 168, "end_line": 174, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L168-L174", "decorators": [], "calls": [ "built_parts.append", "_text_content_part", "text_parts.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_build_string_mllm_message_content", "qualname": "_build_string_mllm_message_content", "full_name": "vllm_mlx.models.mllm._build_string_mllm_message_content", "kind": "function", "signature": "def _build_string_mllm_message_content(content: str, role: str) -> tuple[object, bool]", "parameters": [ { "name": "content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "role", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[object, bool]", "docstring": "", "summary": "Function `_build_string_mllm_message_content` calls `_text_content_part`; has 3 explicit return paths.", "implementation": "Function `_build_string_mllm_message_content` calls `_text_content_part`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 177, "end_line": 182, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L177-L182", "decorators": [], "calls": [ "_text_content_part" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "('', False)", "(content, True)", "([_text_content_part(content)], True)" ], "awaits": false, "yields": false }, { "name": "_append_ordered_mllm_content_part", "qualname": "_append_ordered_mllm_content_part", "full_name": "vllm_mlx.models.mllm._append_ordered_mllm_content_part", "kind": "function", "signature": "def _append_ordered_mllm_content_part(raw_item: object, *, built_parts: list[dict[str, str]], text_parts: list[str], all_image_urls: list[str], video_frame_count: int) -> int", "parameters": [ { "name": "raw_item", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "built_parts", "kind": "keyword-only", "annotation": "list[dict[str, str]]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "text_parts", "kind": "keyword-only", "annotation": "list[str]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "all_image_urls", "kind": "keyword-only", "annotation": "list[str]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "video_frame_count", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "int", "docstring": "", "summary": "Function `_append_ordered_mllm_content_part` calls `_normalize_content_part`, `isinstance`, `_append_text_content_part`, `item.get`; has 2 explicit return paths.", "implementation": "Function `_append_ordered_mllm_content_part` calls `_normalize_content_part`, `isinstance`, `_append_text_content_part`, `item.get`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 185, "end_line": 220, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L185-L220", "decorators": [], "calls": [ "_normalize_content_part", "isinstance", "_append_text_content_part", "item.get", "_extract_media_url", "all_image_urls.append", "built_parts.append", "built_parts.extend", "range" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "video_frame_count", "0" ], "awaits": false, "yields": false }, { "name": "_build_ordered_mllm_message_content", "qualname": "_build_ordered_mllm_message_content", "full_name": "vllm_mlx.models.mllm._build_ordered_mllm_message_content", "kind": "function", "signature": "def _build_ordered_mllm_message_content(content: object, *, role: str, all_image_urls: list[str], video_frame_count: int=0) -> tuple[object, bool]", "parameters": [ { "name": "content", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "role", "kind": "keyword-only", "annotation": "str", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "all_image_urls", "kind": "keyword-only", "annotation": "list[str]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "video_frame_count", "kind": "keyword-only", "annotation": "int", "default": "0", "required": false, "description": "Optional keyword-only input; defaults to `0`." } ], "return_annotation": "tuple[object, bool]", "docstring": "Build template content while preserving OpenAI media/text part order.", "summary": "Build template content while preserving OpenAI media/text part order.", "implementation": "Function `_build_ordered_mllm_message_content` calls `isinstance`, `_build_string_mllm_message_content`, `_append_ordered_mllm_content_part`, `''.join`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 223, "end_line": 254, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L223-L254", "decorators": [], "calls": [ "isinstance", "_build_string_mllm_message_content", "_append_ordered_mllm_content_part", "''.join", "bool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_build_string_mllm_message_content(content, role)", "('', False)", "(text, bool(text))", "(built_parts, bool(built_parts))" ], "awaits": false, "yields": false }, { "name": "_normalize_mllm_tool_calls", "qualname": "_normalize_mllm_tool_calls", "full_name": "vllm_mlx.models.mllm._normalize_mllm_tool_calls", "kind": "function", "signature": "def _normalize_mllm_tool_calls(tool_calls: list) -> list", "parameters": [ { "name": "tool_calls", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list", "docstring": "Normalize replayed assistant tool calls for chat templates.\n\nMirrors ``_normalize_tool_call_arguments_for_template`` in\n``vllm_mlx/engine/batched.py``: JSON argument strings become mappings so\ntemplates that iterate argument keys render correctly.", "summary": "Normalize replayed assistant tool calls for chat templates.", "implementation": "Function `_normalize_mllm_tool_calls` calls `_normalize_content_part`, `normalize_messages_for_chat_template`, `normalized[0].get`; returns `normalized[0].get('tool_calls', plain_calls)`.", "documented": true, "public": false, "addressable": true, "line": 257, "end_line": 268, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L257-L268", "decorators": [], "calls": [ "_normalize_content_part", "normalize_messages_for_chat_template", "normalized[0].get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "normalized[0].get('tool_calls', plain_calls)" ], "awaits": false, "yields": false }, { "name": "_build_mllm_chat_messages", "qualname": "_build_mllm_chat_messages", "full_name": "vllm_mlx.models.mllm._build_mllm_chat_messages", "kind": "function", "signature": "def _build_mllm_chat_messages(messages: list[dict], *, all_image_urls: list[str], video_frame_counts: dict[int, int]) -> list[dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "all_image_urls", "kind": "keyword-only", "annotation": "list[str]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "video_frame_counts", "kind": "keyword-only", "annotation": "dict[int, int]", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "list[dict]", "docstring": "Build chat-template messages without reordering multimodal content parts.", "summary": "Build chat-template messages without reordering multimodal content parts.", "implementation": "Function `_build_mllm_chat_messages` calls `enumerate`, `msg.get`, `isinstance`, `str`; returns `chat_messages`.", "documented": true, "public": false, "addressable": true, "line": 271, "end_line": 315, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L271-L315", "decorators": [], "calls": [ "enumerate", "msg.get", "isinstance", "str", "_build_ordered_mllm_message_content", "video_frame_counts.get", "_normalize_mllm_tool_calls", "chat_messages.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "chat_messages" ], "awaits": false, "yields": false }, { "name": "MultimodalInput", "qualname": "MultimodalInput", "full_name": "vllm_mlx.models.mllm.MultimodalInput", "kind": "class", "signature": "class MultimodalInput", "parameters": [ { "name": "prompt", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "images", "kind": "field", "annotation": "list[str]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "videos", "kind": "field", "annotation": "list[str]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "audio", "kind": "field", "annotation": "list[str]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." } ], "return_annotation": "MultimodalInput", "docstring": "Input for multimodal generation.", "summary": "Input for multimodal generation.", "implementation": "Class `MultimodalInput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 319, "end_line": 325, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L319-L325", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MLLMOutput", "qualname": "MLLMOutput", "full_name": "vllm_mlx.models.mllm.MLLMOutput", "kind": "class", "signature": "class MLLMOutput", "parameters": [ { "name": "text", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "finish_reason", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "completion_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_drafts", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_accepted", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "MLLMOutput", "docstring": "Output from multimodal language model.", "summary": "Output from multimodal language model.", "implementation": "Class `MLLMOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 329, "end_line": 337, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L329-L337", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load_gemma4_assistant_drafter", "qualname": "load_gemma4_assistant_drafter", "full_name": "vllm_mlx.models.mllm.load_gemma4_assistant_drafter", "kind": "function", "signature": "def load_gemma4_assistant_drafter(model_path: str)", "parameters": [ { "name": "model_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Load a Gemma 4 assistant drafter for mlx-vlm speculative decoding.", "summary": "Load a Gemma 4 assistant drafter for mlx-vlm speculative decoding.", "implementation": "Function `load_gemma4_assistant_drafter` calls `ImportError`, `version`, `logger.info`, `Path`; can raise `ImportError`, `FileNotFoundError`; returns `model`.", "documented": true, "public": true, "addressable": true, "line": 340, "end_line": 381, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L340-L381", "decorators": [], "calls": [ "ImportError", "version", "logger.info", "Path", "sorted", "path.glob", "config_path.exists", "FileNotFoundError", "arch.ModelConfig.from_dict", "json.loads", "config_path.read_text", "arch.Model", "weights.update", "mx.load", "str", "hasattr", "model.sanitize", "model.load_weights", "list", "weights.items", "mx.eval", "model.parameters", "model.eval" ], "state_reads": [], "state_writes": [], "raises": [ "ImportError", "FileNotFoundError" ], "return_expressions": [ "model" ], "awaits": false, "yields": false }, { "name": "_count_draft_tokens", "qualname": "_count_draft_tokens", "full_name": "vllm_mlx.models.mllm._count_draft_tokens", "kind": "function", "signature": "def _count_draft_tokens(draft_tokens) -> int", "parameters": [ { "name": "draft_tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Best-effort drafted-token count for an mlx-vlm drafter output.", "summary": "Best-effort drafted-token count for an mlx-vlm drafter output.", "implementation": "Function `_count_draft_tokens` calls `getattr`, `max`, `int`, `len`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 387, "end_line": 398, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L387-L398", "decorators": [], "calls": [ "getattr", "max", "int", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "max(int(shape[-1]), 0)", "max(len(draft_tokens), 0)", "0" ], "awaits": false, "yields": false }, { "name": "_install_draft_metrics_hooks", "qualname": "_install_draft_metrics_hooks", "full_name": "vllm_mlx.models.mllm._install_draft_metrics_hooks", "kind": "function", "signature": "def _install_draft_metrics_hooks(draft_model) -> None", "parameters": [ { "name": "draft_model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Record actual drafted token counts from mlx-vlm assistant drafters.", "summary": "Record actual drafted token counts from mlx-vlm assistant drafters.", "implementation": "Function `_install_draft_metrics_hooks` calls `getattr`, `hasattr`, `callable`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 401, "end_line": 428, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L401-L428", "decorators": [], "calls": [ "getattr", "hasattr", "callable" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "draft_block_with_metrics", "qualname": "_install_draft_metrics_hooks.draft_block_with_metrics", "full_name": "vllm_mlx.models.mllm._install_draft_metrics_hooks.draft_block_with_metrics", "kind": "nested function", "signature": "def draft_block_with_metrics(*args, **kwargs)", "parameters": [ { "name": "*args", "kind": "variadic positional", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic positional inputs accepted by this callable." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_draft_metrics_hooks.draft_block_with_metrics` calls `draft_block`, `draft_model._vllm_mlx_draft_counts.append`, `_count_draft_tokens`; returns `draft_tokens`.", "implementation": "Nested Function `_install_draft_metrics_hooks.draft_block_with_metrics` calls `draft_block`, `draft_model._vllm_mlx_draft_counts.append`, `_count_draft_tokens`; returns `draft_tokens`.", "documented": false, "public": false, "addressable": false, "line": 412, "end_line": 415, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L412-L415", "decorators": [], "calls": [ "draft_block", "draft_model._vllm_mlx_draft_counts.append", "_count_draft_tokens" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "draft_tokens" ], "awaits": false, "yields": false }, { "name": "reset_with_metrics", "qualname": "_install_draft_metrics_hooks.reset_with_metrics", "full_name": "vllm_mlx.models.mllm._install_draft_metrics_hooks.reset_with_metrics", "kind": "nested function", "signature": "def reset_with_metrics(*args, **kwargs)", "parameters": [ { "name": "*args", "kind": "variadic positional", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic positional inputs accepted by this callable." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_draft_metrics_hooks.reset_with_metrics` calls `reset`; returns `reset(*args, **kwargs)`.", "implementation": "Nested Function `_install_draft_metrics_hooks.reset_with_metrics` calls `reset`; returns `reset(*args, **kwargs)`.", "documented": false, "public": false, "addressable": false, "line": 422, "end_line": 424, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L422-L424", "decorators": [], "calls": [ "reset" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "reset(*args, **kwargs)" ], "awaits": false, "yields": false }, { "name": "is_base64_image", "qualname": "is_base64_image", "full_name": "vllm_mlx.models.mllm.is_base64_image", "kind": "function", "signature": "def is_base64_image(s: str) -> bool", "parameters": [ { "name": "s", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if string is base64-encoded image data.", "summary": "Check if string is base64-encoded image data.", "implementation": "Function `is_base64_image` calls `s.startswith`, `len`; returns `s.startswith('data:image/') or (len(s) > 100 and (not s.startswith(('http://', 'https://', '/'))))`.", "documented": true, "public": true, "addressable": true, "line": 431, "end_line": 435, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L431-L435", "decorators": [], "calls": [ "s.startswith", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "s.startswith('data:image/') or (len(s) > 100 and (not s.startswith(('http://', 'https://', '/'))))" ], "awaits": false, "yields": false }, { "name": "is_url", "qualname": "is_url", "full_name": "vllm_mlx.models.mllm.is_url", "kind": "function", "signature": "def is_url(s: str) -> bool", "parameters": [ { "name": "s", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if string is a URL.", "summary": "Check if string is a URL.", "implementation": "Function `is_url` calls `s.startswith`; returns `s.startswith(('http://', 'https://'))`.", "documented": true, "public": true, "addressable": true, "line": 438, "end_line": 440, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L438-L440", "decorators": [], "calls": [ "s.startswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "s.startswith(('http://', 'https://'))" ], "awaits": false, "yields": false }, { "name": "is_base64_video", "qualname": "is_base64_video", "full_name": "vllm_mlx.models.mllm.is_base64_video", "kind": "function", "signature": "def is_base64_video(s: str) -> bool", "parameters": [ { "name": "s", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if string is base64-encoded video data.", "summary": "Check if string is base64-encoded video data.", "implementation": "Function `is_base64_video` calls `s.startswith`; returns `s.startswith('data:video/')`.", "documented": true, "public": true, "addressable": true, "line": 443, "end_line": 445, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L443-L445", "decorators": [], "calls": [ "s.startswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "s.startswith('data:video/')" ], "awaits": false, "yields": false }, { "name": "is_base64_audio", "qualname": "is_base64_audio", "full_name": "vllm_mlx.models.mllm.is_base64_audio", "kind": "function", "signature": "def is_base64_audio(s: str) -> bool", "parameters": [ { "name": "s", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if string is base64-encoded audio data.", "summary": "Check if string is base64-encoded audio data.", "implementation": "Function `is_base64_audio` calls `s.startswith`; returns `s.startswith('data:audio/')`.", "documented": true, "public": true, "addressable": true, "line": 448, "end_line": 450, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L448-L450", "decorators": [], "calls": [ "s.startswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "s.startswith('data:audio/')" ], "awaits": false, "yields": false }, { "name": "decode_base64_image", "qualname": "decode_base64_image", "full_name": "vllm_mlx.models.mllm.decode_base64_image", "kind": "function", "signature": "def decode_base64_image(base64_string: str, max_length: int=MAX_BASE64_IMAGE_LENGTH) -> bytes", "parameters": [ { "name": "base64_string", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Base64 encoded image (optionally with data URL prefix)" }, { "name": "max_length", "kind": "positional or keyword", "annotation": "int", "default": "MAX_BASE64_IMAGE_LENGTH", "required": false, "description": "Maximum allowed length of base64 string" } ], "return_annotation": "bytes", "docstring": "Decode base64 image to bytes.\n\nArgs:\n base64_string: Base64 encoded image (optionally with data URL prefix)\n max_length: Maximum allowed length of base64 string\n\nReturns:\n Decoded image bytes\n\nRaises:\n FileSizeExceededError: If base64 string exceeds max_length", "summary": "Decode base64 image to bytes.", "implementation": "Function `decode_base64_image` calls `len`, `FileSizeExceededError`, `base64_string.startswith`, `base64_string.split`; can raise `FileSizeExceededError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 453, "end_line": 480, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L453-L480", "decorators": [], "calls": [ "len", "FileSizeExceededError", "base64_string.startswith", "base64_string.split", "base64.b64decode" ], "state_reads": [], "state_writes": [], "raises": [ "FileSizeExceededError" ], "return_expressions": [ "base64.b64decode(data)", "base64.b64decode(base64_string)" ], "awaits": false, "yields": false }, { "name": "_validate_url_safety", "qualname": "_validate_url_safety", "full_name": "vllm_mlx.models.mllm._validate_url_safety", "kind": "function", "signature": "def _validate_url_safety(url: str) -> None", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Reject remote URLs that target local or private network resources.", "summary": "Reject remote URLs that target local or private network resources.", "implementation": "Function `_validate_url_safety` calls `urlparse`, `UnsafeRemoteURLError`, `hostname.endswith`, `ipaddress.ip_address`; can raise `UnsafeRemoteURLError`.", "documented": true, "public": false, "addressable": true, "line": 483, "end_line": 519, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L483-L519", "decorators": [], "calls": [ "urlparse", "UnsafeRemoteURLError", "hostname.endswith", "ipaddress.ip_address", "socket.getaddrinfo", "str", "', '.join", "sorted", "set" ], "state_reads": [], "state_writes": [], "raises": [ "UnsafeRemoteURLError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_request_with_safe_redirects", "qualname": "_request_with_safe_redirects", "full_name": "vllm_mlx.models.mllm._request_with_safe_redirects", "kind": "function", "signature": "def _request_with_safe_redirects(method: str, url: str, *, timeout: int, headers: dict[str, str], stream: bool=False, max_redirects: int=5)", "parameters": [ { "name": "method", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "headers", "kind": "keyword-only", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "stream", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." }, { "name": "max_redirects", "kind": "keyword-only", "annotation": "int", "default": "5", "required": false, "description": "Optional keyword-only input; defaults to `5`." } ], "return_annotation": "not annotated", "docstring": "Issue a requests call while validating every redirect target.", "summary": "Issue a requests call while validating every redirect target.", "implementation": "Function `_request_with_safe_redirects` calls `range`, `_validate_url_safety`, `requests.request`, `response.headers.get`; can raise `UnsafeRemoteURLError`; returns `response`.", "documented": true, "public": false, "addressable": true, "line": 522, "end_line": 557, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L522-L557", "decorators": [], "calls": [ "range", "_validate_url_safety", "requests.request", "response.headers.get", "response.close", "UnsafeRemoteURLError", "urljoin" ], "state_reads": [], "state_writes": [], "raises": [ "UnsafeRemoteURLError" ], "return_expressions": [ "response" ], "awaits": false, "yields": false }, { "name": "download_image", "qualname": "download_image", "full_name": "vllm_mlx.models.mllm.download_image", "kind": "function", "signature": "def download_image(url: str, timeout: int=30, max_size: int=MAX_IMAGE_SIZE) -> str", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Image URL" }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "30", "required": false, "description": "Download timeout in seconds" }, { "name": "max_size", "kind": "positional or keyword", "annotation": "int", "default": "MAX_IMAGE_SIZE", "required": false, "description": "Maximum allowed file size in bytes" } ], "return_annotation": "str", "docstring": "Download image from URL and return local path.\n\nArgs:\n url: Image URL\n timeout: Download timeout in seconds\n max_size: Maximum allowed file size in bytes\n\nReturns:\n Local file path to downloaded image\n\nRaises:\n FileSizeExceededError: If image exceeds max_size", "summary": "Download image from URL and return local path.", "implementation": "Function `download_image` calls `_request_with_safe_redirects`, `head_response.headers.get`, `int`, `FileSizeExceededError`; can raise `FileSizeExceededError`; returns `_temp_manager.register(temp_file.name)`.", "documented": true, "public": true, "addressable": true, "line": 560, "end_line": 645, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L560-L645", "decorators": [], "calls": [ "_request_with_safe_redirects", "head_response.headers.get", "int", "FileSizeExceededError", "response.raise_for_status", "response.headers.get", "urlparse", "Path", "tempfile.NamedTemporaryFile", "response.iter_content", "len", "temp_file.close", "os.unlink", "temp_file.write", "os.path.exists", "_temp_manager.register" ], "state_reads": [], "state_writes": [], "raises": [ "FileSizeExceededError" ], "return_expressions": [ "_temp_manager.register(temp_file.name)" ], "awaits": false, "yields": false }, { "name": "_download_media", "qualname": "_download_media", "full_name": "vllm_mlx.models.mllm._download_media", "kind": "function", "signature": "def _download_media(url: str, media_type: str, ext_map: dict[str, str], default_ext: str, timeout: int, max_size: int) -> str", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "media_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "ext_map", "kind": "positional or keyword", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "default_ext", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Download media from URL, enforce size limits, and return a local temp path.", "summary": "Download media from URL, enforce size limits, and return a local temp path.", "implementation": "Function `_download_media` calls `logger.info`, `_request_with_safe_redirects`, `head_response.headers.get`, `int`; can raise `FileSizeExceededError`; returns `_temp_manager.register(temp_file.name)`.", "documented": true, "public": false, "addressable": true, "line": 670, "end_line": 748, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L670-L748", "decorators": [], "calls": [ "logger.info", "_request_with_safe_redirects", "head_response.headers.get", "int", "FileSizeExceededError", "media_type.capitalize", "response.raise_for_status", "response.headers.get", "response.headers.get('content-type', '').lower", "ext_map.items", "Path", "urlparse", "tempfile.NamedTemporaryFile", "response.iter_content", "len", "temp_file.close", "os.unlink", "temp_file.write", "os.path.exists", "Path(temp_file.name).stat", "_temp_manager.register" ], "state_reads": [], "state_writes": [], "raises": [ "FileSizeExceededError" ], "return_expressions": [ "_temp_manager.register(temp_file.name)" ], "awaits": false, "yields": false }, { "name": "download_video", "qualname": "download_video", "full_name": "vllm_mlx.models.mllm.download_video", "kind": "function", "signature": "def download_video(url: str, timeout: int=120, max_size: int=MAX_VIDEO_SIZE) -> str", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "120", "required": false, "description": "Optional positional or keyword input; defaults to `120`." }, { "name": "max_size", "kind": "positional or keyword", "annotation": "int", "default": "MAX_VIDEO_SIZE", "required": false, "description": "Optional positional or keyword input; defaults to `MAX_VIDEO_SIZE`." } ], "return_annotation": "str", "docstring": "Download video from URL and return local path.", "summary": "Download video from URL and return local path.", "implementation": "Function `download_video` calls `_download_media`; returns `_download_media(url, 'video', _VIDEO_EXT_MAP, '.mp4', timeout, max_size)`.", "documented": true, "public": true, "addressable": true, "line": 751, "end_line": 753, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L751-L753", "decorators": [], "calls": [ "_download_media" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_download_media(url, 'video', _VIDEO_EXT_MAP, '.mp4', timeout, max_size)" ], "awaits": false, "yields": false }, { "name": "download_audio", "qualname": "download_audio", "full_name": "vllm_mlx.models.mllm.download_audio", "kind": "function", "signature": "def download_audio(url: str, timeout: int=120, max_size: int=MAX_AUDIO_SIZE) -> str", "parameters": [ { "name": "url", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "int", "default": "120", "required": false, "description": "Optional positional or keyword input; defaults to `120`." }, { "name": "max_size", "kind": "positional or keyword", "annotation": "int", "default": "MAX_AUDIO_SIZE", "required": false, "description": "Optional positional or keyword input; defaults to `MAX_AUDIO_SIZE`." } ], "return_annotation": "str", "docstring": "Download audio from URL and return local path.", "summary": "Download audio from URL and return local path.", "implementation": "Function `download_audio` calls `_download_media`; returns `_download_media(url, 'audio', _AUDIO_EXT_MAP, '.wav', timeout, max_size)`.", "documented": true, "public": true, "addressable": true, "line": 756, "end_line": 758, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L756-L758", "decorators": [], "calls": [ "_download_media" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_download_media(url, 'audio', _AUDIO_EXT_MAP, '.wav', timeout, max_size)" ], "awaits": false, "yields": false }, { "name": "decode_base64_video", "qualname": "decode_base64_video", "full_name": "vllm_mlx.models.mllm.decode_base64_video", "kind": "function", "signature": "def decode_base64_video(base64_string: str, max_length: int=MAX_BASE64_VIDEO_LENGTH) -> str", "parameters": [ { "name": "base64_string", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Base64-encoded video with data URL prefix" }, { "name": "max_length", "kind": "positional or keyword", "annotation": "int", "default": "MAX_BASE64_VIDEO_LENGTH", "required": false, "description": "Maximum allowed length of base64 string" } ], "return_annotation": "str", "docstring": "Decode base64 video to temp file and return path.\n\nSupports format: data:video/mp4;base64,AAAA...\n\nArgs:\n base64_string: Base64-encoded video with data URL prefix\n max_length: Maximum allowed length of base64 string\n\nReturns:\n Local file path to decoded video\n\nRaises:\n FileSizeExceededError: If base64 string exceeds max_length", "summary": "Decode base64 video to temp file and return path.", "implementation": "Function `decode_base64_video` calls `len`, `FileSizeExceededError`, `base64_string.startswith`, `base64_string.split`; can raise `FileSizeExceededError`; returns `_temp_manager.register(temp_file.name)`.", "documented": true, "public": true, "addressable": true, "line": 761, "end_line": 807, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L761-L807", "decorators": [], "calls": [ "len", "FileSizeExceededError", "base64_string.startswith", "base64_string.split", "header.split", "format_part.split", "base64.b64decode", "tempfile.NamedTemporaryFile", "temp_file.write", "temp_file.close", "logger.info", "_temp_manager.register" ], "state_reads": [], "state_writes": [], "raises": [ "FileSizeExceededError" ], "return_expressions": [ "_temp_manager.register(temp_file.name)" ], "awaits": false, "yields": false }, { "name": "decode_base64_audio", "qualname": "decode_base64_audio", "full_name": "vllm_mlx.models.mllm.decode_base64_audio", "kind": "function", "signature": "def decode_base64_audio(base64_string: str, max_length: int=MAX_BASE64_AUDIO_LENGTH) -> str", "parameters": [ { "name": "base64_string", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_length", "kind": "positional or keyword", "annotation": "int", "default": "MAX_BASE64_AUDIO_LENGTH", "required": false, "description": "Optional positional or keyword input; defaults to `MAX_BASE64_AUDIO_LENGTH`." } ], "return_annotation": "str", "docstring": "Decode base64 audio to temp file and return path.\n\nSupports format: data:audio/wav;base64,AAAA...", "summary": "Decode base64 audio to temp file and return path.", "implementation": "Function `decode_base64_audio` calls `len`, `FileSizeExceededError`, `base64_string.startswith`, `base64_string.split`; can raise `FileSizeExceededError`; returns `_temp_manager.register(temp_file.name)`.", "documented": true, "public": true, "addressable": true, "line": 810, "end_line": 836, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L810-L836", "decorators": [], "calls": [ "len", "FileSizeExceededError", "base64_string.startswith", "base64_string.split", "header.split", "format_part.split", "base64.b64decode", "tempfile.NamedTemporaryFile", "temp_file.write", "temp_file.close", "_temp_manager.register" ], "state_reads": [], "state_writes": [], "raises": [ "FileSizeExceededError" ], "return_expressions": [ "_temp_manager.register(temp_file.name)" ], "awaits": false, "yields": false }, { "name": "process_video_input", "qualname": "process_video_input", "full_name": "vllm_mlx.models.mllm.process_video_input", "kind": "function", "signature": "def process_video_input(video: str | dict) -> str", "parameters": [ { "name": "video", "kind": "positional or keyword", "annotation": "str | dict", "default": "", "required": true, "description": "Video input in any supported format" } ], "return_annotation": "str", "docstring": "Process video input in various formats and return local path.\n\nSupports:\n- URL (http/https)\n- Base64 encoded string (data:video/mp4;base64,...)\n- OpenAI format dict: {\"url\": \"...\"} or {\"url\": \"data:video/...;base64,...\"}\n\nArgs:\n video: Video input in any supported format\n\nReturns:\n Local file path to video", "summary": "Process video input in various formats and return local path.", "implementation": "Function `process_video_input` calls `isinstance`, `video.get`, `url.get`, `ValueError`; can raise `ValueError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 839, "end_line": 874, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L839-L874", "decorators": [], "calls": [ "isinstance", "video.get", "url.get", "ValueError", "is_url", "download_video", "is_base64_video", "decode_base64_video" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "download_video(video)", "decode_base64_video(video)" ], "awaits": false, "yields": false }, { "name": "process_audio_input", "qualname": "process_audio_input", "full_name": "vllm_mlx.models.mllm.process_audio_input", "kind": "function", "signature": "def process_audio_input(audio: str | dict) -> str", "parameters": [ { "name": "audio", "kind": "positional or keyword", "annotation": "str | dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Process audio input in various formats and return local path.\n\nSupports:\n- Local file path\n- URL (http/https)\n- Base64 encoded string (data:audio/wav;base64,...)\n- OpenAI format dict: {\"url\": \"...\"} or {\"audio_url\": {\"url\": \"...\"}}", "summary": "Process audio input in various formats and return local path.", "implementation": "Function `process_audio_input` calls `isinstance`, `audio.get`, `url.get`, `ValueError`; can raise `ValueError`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 877, "end_line": 905, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L877-L905", "decorators": [], "calls": [ "isinstance", "audio.get", "url.get", "ValueError", "is_base64_audio", "decode_base64_audio", "is_url", "download_audio", "len", "Path(audio).exists", "Path" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "decode_base64_audio(audio)", "download_audio(audio)", "audio" ], "awaits": false, "yields": false }, { "name": "_video_has_audio_track", "qualname": "_video_has_audio_track", "full_name": "vllm_mlx.models.mllm._video_has_audio_track", "kind": "function", "signature": "def _video_has_audio_track(video_path: str) -> bool", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True if ffprobe finds an audio stream in the video.", "summary": "Return True if ffprobe finds an audio stream in the video.", "implementation": "Function `_video_has_audio_track` calls `shutil.which`, `subprocess.run`, `bool`, `r.stdout.strip`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 908, "end_line": 935, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L908-L935", "decorators": [], "calls": [ "shutil.which", "subprocess.run", "bool", "r.stdout.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "bool(r.stdout.strip())" ], "awaits": false, "yields": false }, { "name": "_model_has_sound_encoder", "qualname": "_model_has_sound_encoder", "full_name": "vllm_mlx.models.mllm._model_has_sound_encoder", "kind": "function", "signature": "def _model_has_sound_encoder(model) -> bool", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Whether a loaded model exposes a usable sound encoder.\n\nUses ``getattr(..., None) is not None`` rather than ``hasattr`` so model\nwrappers that declare ``sound_encoder`` in ``__init__`` but leave it as\n``None`` until the first encoder pass are correctly treated as not yet\nenabled. A bare ``hasattr`` check would spuriously enable A/V fusion\nagainst a missing encoder and crash the processor downstream.", "summary": "Whether a loaded model exposes a usable sound encoder.", "implementation": "Function `_model_has_sound_encoder` calls `getattr`; returns `getattr(model, 'sound_encoder', None) is not None`.", "documented": true, "public": false, "addressable": true, "line": 938, "end_line": 947, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L938-L947", "decorators": [], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "getattr(model, 'sound_encoder', None) is not None" ], "awaits": false, "yields": false }, { "name": "extract_audio_from_video", "qualname": "extract_audio_from_video", "full_name": "vllm_mlx.models.mllm.extract_audio_from_video", "kind": "function", "signature": "def extract_audio_from_video(video_path: str) -> str | None", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "Extract the audio track from a video file as 16 kHz mono WAV.\n\nReturns the path to the WAV (registered with the temp manager so it's\ncleaned up automatically), or None if the video has no audio or ffmpeg\nis unavailable.", "summary": "Extract the audio track from a video file as 16 kHz mono WAV.", "implementation": "Function `extract_audio_from_video` calls `shutil.which`, `logger.warning`, `_video_has_audio_track`, `tempfile.mkstemp`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 950, "end_line": 1005, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L950-L1005", "decorators": [], "calls": [ "shutil.which", "logger.warning", "_video_has_audio_track", "tempfile.mkstemp", "os.close", "subprocess.run", "os.path.getsize", "os.unlink", "_temp_manager.register" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "_temp_manager.register(out_path)" ], "awaits": false, "yields": false }, { "name": "save_base64_image", "qualname": "save_base64_image", "full_name": "vllm_mlx.models.mllm.save_base64_image", "kind": "function", "signature": "def save_base64_image(base64_string: str) -> str", "parameters": [ { "name": "base64_string", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Save base64 image to temp file and return path. Caches identical images.", "summary": "Save base64 image to temp file and return path.", "implementation": "Function `save_base64_image` calls `hashlib.sha256(base64_string.encode()).hexdigest`, `hashlib.sha256`, `base64_string.encode`, `Path(cached_path).exists`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1012, "end_line": 1048, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1012-L1048", "decorators": [], "calls": [ "hashlib.sha256(base64_string.encode()).hexdigest", "hashlib.sha256", "base64_string.encode", "Path(cached_path).exists", "Path", "decode_base64_image", "tempfile.NamedTemporaryFile", "temp_file.write", "temp_file.close", "_temp_manager.register" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cached_path", "path" ], "awaits": false, "yields": false }, { "name": "process_image_input", "qualname": "process_image_input", "full_name": "vllm_mlx.models.mllm.process_image_input", "kind": "function", "signature": "def process_image_input(image: str | dict) -> str", "parameters": [ { "name": "image", "kind": "positional or keyword", "annotation": "str | dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Process image input in various formats and return local path.\n\nSupports:\n- URL (http/https)\n- Base64 encoded string\n- OpenAI format dict: {\"url\": \"...\"} or {\"url\": \"data:image/...;base64,...\"}", "summary": "Process image input in various formats and return local path.", "implementation": "Function `process_image_input` calls `isinstance`, `image.get`, `url.get`, `ValueError`; can raise `ValueError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1051, "end_line": 1080, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1051-L1080", "decorators": [], "calls": [ "isinstance", "image.get", "url.get", "ValueError", "is_base64_image", "save_base64_image", "is_url", "download_image" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "save_base64_image(image)", "download_image(image)" ], "awaits": false, "yields": false }, { "name": "round_by_factor", "qualname": "round_by_factor", "full_name": "vllm_mlx.models.mllm.round_by_factor", "kind": "function", "signature": "def round_by_factor(x: int, factor: int) -> int", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "factor", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Round to nearest multiple of factor.", "summary": "Round to nearest multiple of factor.", "implementation": "Function `round_by_factor` calls `round`; returns `round(x / factor) * factor`.", "documented": true, "public": true, "addressable": true, "line": 1083, "end_line": 1085, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1083-L1085", "decorators": [], "calls": [ "round" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "round(x / factor) * factor" ], "awaits": false, "yields": false }, { "name": "ceil_by_factor", "qualname": "ceil_by_factor", "full_name": "vllm_mlx.models.mllm.ceil_by_factor", "kind": "function", "signature": "def ceil_by_factor(x: float, factor: int) -> int", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "factor", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Ceiling to next multiple of factor.", "summary": "Ceiling to next multiple of factor.", "implementation": "Function `ceil_by_factor` calls `math.ceil`; returns `math.ceil(x / factor) * factor`.", "documented": true, "public": true, "addressable": true, "line": 1088, "end_line": 1090, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1088-L1090", "decorators": [], "calls": [ "math.ceil" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "math.ceil(x / factor) * factor" ], "awaits": false, "yields": false }, { "name": "floor_by_factor", "qualname": "floor_by_factor", "full_name": "vllm_mlx.models.mllm.floor_by_factor", "kind": "function", "signature": "def floor_by_factor(x: float, factor: int) -> int", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "factor", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Floor to previous multiple of factor.", "summary": "Floor to previous multiple of factor.", "implementation": "Function `floor_by_factor` calls `math.floor`; returns `math.floor(x / factor) * factor`.", "documented": true, "public": true, "addressable": true, "line": 1093, "end_line": 1095, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1093-L1095", "decorators": [], "calls": [ "math.floor" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "math.floor(x / factor) * factor" ], "awaits": false, "yields": false }, { "name": "smart_nframes", "qualname": "smart_nframes", "full_name": "vllm_mlx.models.mllm.smart_nframes", "kind": "function", "signature": "def smart_nframes(total_frames: int, video_fps: float, target_fps: float=DEFAULT_FPS, min_frames: int=MIN_FRAMES, max_frames: int=MAX_FRAMES) -> int", "parameters": [ { "name": "total_frames", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_fps", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "target_fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "Optional positional or keyword input; defaults to `DEFAULT_FPS`." }, { "name": "min_frames", "kind": "positional or keyword", "annotation": "int", "default": "MIN_FRAMES", "required": false, "description": "Optional positional or keyword input; defaults to `MIN_FRAMES`." }, { "name": "max_frames", "kind": "positional or keyword", "annotation": "int", "default": "MAX_FRAMES", "required": false, "description": "Optional positional or keyword input; defaults to `MAX_FRAMES`." } ], "return_annotation": "int", "docstring": "Calculate optimal number of frames to extract from video.\n\nUses smart sampling based on video length and target FPS.", "summary": "Calculate optimal number of frames to extract from video.", "implementation": "Function `smart_nframes` calls `max`, `min`, `floor_by_factor`, `int`; returns `int(nframes)`.", "documented": true, "public": true, "addressable": true, "line": 1098, "end_line": 1120, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1098-L1120", "decorators": [], "calls": [ "max", "min", "floor_by_factor", "int" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "int(nframes)" ], "awaits": false, "yields": false }, { "name": "extract_video_frames_smart", "qualname": "extract_video_frames_smart", "full_name": "vllm_mlx.models.mllm.extract_video_frames_smart", "kind": "function", "signature": "def extract_video_frames_smart(video_path: str, fps: float=DEFAULT_FPS, max_frames: int=MAX_FRAMES, resize: tuple[int, int] | None=None) -> list[np.ndarray]", "parameters": [ { "name": "video_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Path to video file" }, { "name": "fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "Target frames per second (default: 2.0)" }, { "name": "max_frames", "kind": "positional or keyword", "annotation": "int", "default": "MAX_FRAMES", "required": false, "description": "Maximum frames to extract" }, { "name": "resize", "kind": "positional or keyword", "annotation": "tuple[int, int] | None", "default": "None", "required": false, "description": "Optional (width, height) to resize frames" } ], "return_annotation": "list[np.ndarray]", "docstring": "Extract frames from video with smart sampling.\n\nArgs:\n video_path: Path to video file\n fps: Target frames per second (default: 2.0)\n max_frames: Maximum frames to extract\n resize: Optional (width, height) to resize frames\n\nReturns:\n List of frame arrays (RGB format)", "summary": "Extract frames from video with smart sampling.", "implementation": "Function `extract_video_frames_smart` calls `ImportError`, `cv2.VideoCapture`, `cap.isOpened`, `ValueError`; can raise `ImportError`, `ValueError`; returns `frames`.", "documented": true, "public": true, "addressable": true, "line": 1123, "end_line": 1187, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1123-L1187", "decorators": [], "calls": [ "ImportError", "cv2.VideoCapture", "cap.isOpened", "ValueError", "int", "cap.get", "smart_nframes", "np.linspace(0, total_frames - 1, nframes).round().astype", "np.linspace(0, total_frames - 1, nframes).round", "np.linspace", "logger.info", "cap.set", "cap.read", "cv2.cvtColor", "cv2.resize", "frames.append", "cap.release" ], "state_reads": [], "state_writes": [], "raises": [ "ImportError", "ValueError" ], "return_expressions": [ "frames" ], "awaits": false, "yields": false }, { "name": "save_frames_to_temp", "qualname": "save_frames_to_temp", "full_name": "vllm_mlx.models.mllm.save_frames_to_temp", "kind": "function", "signature": "def save_frames_to_temp(frames: list[np.ndarray]) -> list[str]", "parameters": [ { "name": "frames", "kind": "positional or keyword", "annotation": "list[np.ndarray]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Save frame arrays to temporary files and return paths.", "summary": "Save frame arrays to temporary files and return paths.", "implementation": "Function `save_frames_to_temp` calls `ImportError`, `enumerate`, `Image.fromarray`, `tempfile.NamedTemporaryFile`; can raise `ImportError`; returns `paths`.", "documented": true, "public": true, "addressable": true, "line": 1190, "end_line": 1204, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1190-L1204", "decorators": [], "calls": [ "ImportError", "enumerate", "Image.fromarray", "tempfile.NamedTemporaryFile", "img.save", "paths.append", "_temp_manager.register" ], "state_reads": [], "state_writes": [], "raises": [ "ImportError" ], "return_expressions": [ "paths" ], "awaits": false, "yields": false }, { "name": "MLXMultimodalLM", "qualname": "MLXMultimodalLM", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM", "kind": "class", "signature": "class MLXMultimodalLM", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "enable_cache", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Enable KV cache for repeated image/video+prompt (default: True)" }, { "name": "cache_size", "kind": "positional or keyword", "annotation": "int", "default": "50", "required": false, "description": "Maximum cache entries (default: 50)" }, { "name": "max_kv_size", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Maximum KV cache size per sequence (0 = unbounded)" }, { "name": "draft_model", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional MLLM speculative draft/assistant model path." }, { "name": "draft_kind", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional mlx-vlm draft kind, for example \"mtp\"." }, { "name": "draft_block_size", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional speculative block size passed to mlx-vlm." } ], "return_annotation": "MLXMultimodalLM", "docstring": "Wrapper around mlx-vlm for multimodal inference.\n\nThis class provides a unified interface for multimodal language models\nusing Apple's MLX framework. Supports:\n- Image understanding (single and multi-image)\n- Video understanding (smart frame extraction)\n- Audio understanding (for supported models)\n- OpenAI-compatible API format\n\nSupported models include:\n- Qwen2-VL / Qwen2.5-VL / Qwen3-VL\n- LLaVA\n- Idefics3\n- PaliGemma\n- And more via mlx-vlm\n\nExample:\n >>> model = MLXMultimodalLM(\"mlx-community/Qwen2-VL-2B-Instruct-4bit\")\n >>> model.load()\n >>> output = model.generate(\n ... prompt=\"What's in this image?\",\n ... images=[\"photo.jpg\"]\n ... )\n >>> print(output.text)", "summary": "Wrapper around mlx-vlm for multimodal inference.", "implementation": "Class `MLXMultimodalLM` declares 29 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 1207, "end_line": 2938, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1207-L2938", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLXMultimodalLM.__init__", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, trust_remote_code: bool=False, enable_cache: bool=True, cache_size: int=50, max_kv_size: int=0, draft_model: str | None=None, draft_kind: str | None=None, draft_block_size: int | None=None)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether to trust remote code" }, { "name": "enable_cache", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Enable KV cache for repeated image/video+prompt (default: True)" }, { "name": "cache_size", "kind": "positional or keyword", "annotation": "int", "default": "50", "required": false, "description": "Maximum cache entries (default: 50)" }, { "name": "max_kv_size", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Maximum KV cache size per sequence (0 = unbounded)" }, { "name": "draft_model", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional MLLM speculative draft/assistant model path." }, { "name": "draft_kind", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional mlx-vlm draft kind, for example \"mtp\"." }, { "name": "draft_block_size", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional speculative block size passed to mlx-vlm." } ], "return_annotation": "not annotated", "docstring": "Initialize the MLX multimodal language model.\n\nArgs:\n model_name: HuggingFace model name or local path\n trust_remote_code: Whether to trust remote code\n enable_cache: Enable KV cache for repeated image/video+prompt (default: True)\n cache_size: Maximum cache entries (default: 50)\n max_kv_size: Maximum KV cache size per sequence (0 = unbounded)\n draft_model: Optional MLLM speculative draft/assistant model path.\n draft_kind: Optional mlx-vlm draft kind, for example \"mtp\".\n draft_block_size: Optional speculative block size passed to mlx-vlm.", "summary": "Initialize the MLX multimodal language model.", "implementation": "Method `MLXMultimodalLM.__init__` updates `self.model_name`, `self.trust_remote_code`, `self.enable_cache`, `self.max_kv_size`; calls `MLLMPrefixCacheManager`.", "documented": true, "public": false, "addressable": true, "line": 1235, "end_line": 1278, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1235-L1278", "decorators": [], "calls": [ "MLLMPrefixCacheManager" ], "state_reads": [], "state_writes": [ "self.model_name", "self.trust_remote_code", "self.enable_cache", "self.max_kv_size", "self.draft_model_path", "self.draft_kind", "self.draft_block_size", "self.model", "self.processor", "self.config", "self._draft_model", "self._loaded", "self._video_native", "self._video_native_with_audio", "self._cache_manager" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load", "qualname": "MLXMultimodalLM.load", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.load", "kind": "method", "signature": "def load(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the model and processor.", "summary": "Load the model and processor.", "implementation": "Method `MLXMultimodalLM.load` updates `self.model`, `self.processor`, `self.config`, `self._draft_model`; calls `logger.info`, `load`, `load_config`, `self._load_draft_model`; can raise `ImportError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 1280, "end_line": 1323, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1280-L1323", "decorators": [], "calls": [ "logger.info", "load", "load_config", "self._load_draft_model", "_install_draft_metrics_hooks", "hasattr", "_model_has_sound_encoder", "ImportError", "logger.error" ], "state_reads": [ "self._loaded", "self.model_name", "self.draft_model_path", "self._load_draft_model", "self._draft_model", "self.model.config", "self.model", "self._video_native", "self._video_native_with_audio" ], "state_writes": [ "self.model", "self.processor", "self.config", "self._draft_model", "self._loaded", "self._video_native", "self._video_native_with_audio" ], "raises": [ "ImportError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_load_draft_model", "qualname": "MLXMultimodalLM._load_draft_model", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._load_draft_model", "kind": "method", "signature": "def _load_draft_model(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `MLXMultimodalLM._load_draft_model` calls `load_gemma4_assistant_drafter`, `load`; has 2 explicit return paths.", "implementation": "Method `MLXMultimodalLM._load_draft_model` calls `load_gemma4_assistant_drafter`, `load`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1325, "end_line": 1332, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1325-L1332", "decorators": [], "calls": [ "load_gemma4_assistant_drafter", "load" ], "state_reads": [ "self.draft_kind", "self.draft_model_path" ], "state_writes": [], "raises": [], "return_expressions": [ "load_gemma4_assistant_drafter(self.draft_model_path)", "draft_model" ], "awaits": false, "yields": false }, { "name": "_draft_generation_kwargs", "qualname": "MLXMultimodalLM._draft_generation_kwargs", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._draft_generation_kwargs", "kind": "method", "signature": "def _draft_generation_kwargs(self, call_kwargs: dict | None=None) -> dict", "parameters": [ { "name": "call_kwargs", "kind": "positional or keyword", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict", "docstring": "Return mlx-vlm drafter kwargs when the request explicitly opts in.\n\n``call_kwargs`` is the outbound mlx-vlm kwargs dict. This method removes\nvllm-mlx drafter control keys before the dict is forwarded so caller\npassthrough values cannot conflict with the configured server drafter.", "summary": "Return mlx-vlm drafter kwargs when the request explicitly opts in.", "implementation": "Method `MLXMultimodalLM._draft_generation_kwargs` calls `bool`, `call_kwargs.pop`, `_install_draft_metrics_hooks`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1334, "end_line": 1355, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1334-L1355", "decorators": [], "calls": [ "bool", "call_kwargs.pop", "_install_draft_metrics_hooks" ], "state_reads": [ "self._draft_model", "self.draft_kind", "self.draft_block_size" ], "state_writes": [], "raises": [], "return_expressions": [ "{}", "kwargs" ], "awaits": false, "yields": false }, { "name": "_reset_draft_metrics", "qualname": "MLXMultimodalLM._reset_draft_metrics", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._reset_draft_metrics", "kind": "method", "signature": "def _reset_draft_metrics(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "", "summary": "Method `MLXMultimodalLM._reset_draft_metrics` updates `self._draft_model.accept_lens`, `self._draft_model._vllm_mlx_draft_counts`; calls `hasattr`; returns `0`.", "implementation": "Method `MLXMultimodalLM._reset_draft_metrics` updates `self._draft_model.accept_lens`, `self._draft_model._vllm_mlx_draft_counts`; calls `hasattr`; returns `0`.", "documented": false, "public": false, "addressable": true, "line": 1357, "end_line": 1364, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1357-L1364", "decorators": [], "calls": [ "hasattr" ], "state_reads": [ "self._draft_model" ], "state_writes": [ "self._draft_model.accept_lens", "self._draft_model._vllm_mlx_draft_counts" ], "raises": [], "return_expressions": [ "0" ], "awaits": false, "yields": false }, { "name": "_draft_metrics_since", "qualname": "MLXMultimodalLM._draft_metrics_since", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._draft_metrics_since", "kind": "method", "signature": "def _draft_metrics_since(self, start_accept_lens: int) -> dict[str, int]", "parameters": [ { "name": "start_accept_lens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, int]", "docstring": "", "summary": "Method `MLXMultimodalLM._draft_metrics_since` calls `list`, `getattr`, `len`, `int`; has 2 explicit return paths.", "implementation": "Method `MLXMultimodalLM._draft_metrics_since` calls `list`, `getattr`, `len`, `int`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1366, "end_line": 1395, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1366-L1395", "decorators": [], "calls": [ "list", "getattr", "len", "int", "max", "sum" ], "state_reads": [ "self._draft_model", "self.draft_block_size" ], "state_writes": [], "raises": [], "return_expressions": [ "{'mtp_drafts': 0, 'mtp_accepted': 0}", "{'mtp_drafts': mtp_drafts, 'mtp_accepted': sum((int(value) for value in new_accept_lens))}" ], "awaits": false, "yields": false }, { "name": "get_language_model", "qualname": "MLXMultimodalLM.get_language_model", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.get_language_model", "kind": "method", "signature": "def get_language_model(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Extract the underlying language model for mlx_lm TextModel construction.", "summary": "Extract the underlying language model for mlx_lm TextModel construction.", "implementation": "Method `MLXMultimodalLM.get_language_model` returns `self.model.language_model`.", "documented": true, "public": true, "addressable": true, "line": 1397, "end_line": 1399, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1397-L1399", "decorators": [], "calls": [], "state_reads": [ "self.model.language_model", "self.model" ], "state_writes": [], "raises": [], "return_expressions": [ "self.model.language_model" ], "awaits": false, "yields": false }, { "name": "get_tokenizer", "qualname": "MLXMultimodalLM.get_tokenizer", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.get_tokenizer", "kind": "method", "signature": "def get_tokenizer(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Get the text tokenizer (not the multimodal processor).", "summary": "Get the text tokenizer (not the multimodal processor).", "implementation": "Method `MLXMultimodalLM.get_tokenizer` returns `self.processor.tokenizer`.", "documented": true, "public": true, "addressable": true, "line": 1401, "end_line": 1403, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1401-L1403", "decorators": [], "calls": [], "state_reads": [ "self.processor.tokenizer", "self.processor" ], "state_writes": [], "raises": [], "return_expressions": [ "self.processor.tokenizer" ], "awaits": false, "yields": false }, { "name": "_prepare_images", "qualname": "MLXMultimodalLM._prepare_images", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._prepare_images", "kind": "method", "signature": "def _prepare_images(self, images: list) -> list[str]", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Process remote/base64 image inputs into local temp file paths.", "summary": "Process remote/base64 image inputs into local temp file paths.", "implementation": "Method `MLXMultimodalLM._prepare_images` calls `process_image_input`, `processed.append`, `logger.warning`; returns `processed`.", "documented": true, "public": false, "addressable": true, "line": 1405, "end_line": 1414, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1405-L1414", "decorators": [], "calls": [ "process_image_input", "processed.append", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "processed" ], "awaits": false, "yields": false }, { "name": "_prepare_audio", "qualname": "MLXMultimodalLM._prepare_audio", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._prepare_audio", "kind": "method", "signature": "def _prepare_audio(self, audio_inputs: list) -> list[str]", "parameters": [ { "name": "audio_inputs", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Process audio inputs and return local file paths.", "summary": "Process audio inputs and return local file paths.", "implementation": "Method `MLXMultimodalLM._prepare_audio` calls `process_audio_input`, `processed.append`, `logger.warning`; returns `processed`.", "documented": true, "public": false, "addressable": true, "line": 1416, "end_line": 1425, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1416-L1425", "decorators": [], "calls": [ "process_audio_input", "processed.append", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "processed" ], "awaits": false, "yields": false }, { "name": "_prepare_video", "qualname": "MLXMultimodalLM._prepare_video", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._prepare_video", "kind": "method", "signature": "def _prepare_video(self, video_input: str | dict, fps: float=DEFAULT_FPS, max_frames: int=MAX_FRAMES, resolved_path: str | None=None) -> list[str]", "parameters": [ { "name": "video_input", "kind": "positional or keyword", "annotation": "str | dict", "default": "", "required": true, "description": "Video in any supported format" }, { "name": "fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "Frames per second to extract" }, { "name": "max_frames", "kind": "positional or keyword", "annotation": "int", "default": "MAX_FRAMES", "required": false, "description": "Maximum frames to extract" }, { "name": "resolved_path", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional pre-resolved local path. Callers that already ran process_video_input (e.g. for parallel audio extraction) pass it here to avoid re-downloading / re-decoding." } ], "return_annotation": "list[str]", "docstring": "Process video input and extract frames.\n\nSupports:\n- URLs (http/https) - will be downloaded\n- Base64 encoded videos (data:video/mp4;base64,...)\n- OpenAI format dicts: {\"url\": \"...\"} or {\"video_url\": {\"url\": \"...\"}}\n\nArgs:\n video_input: Video in any supported format\n fps: Frames per second to extract\n max_frames: Maximum frames to extract\n resolved_path: Optional pre-resolved local path. Callers that\n already ran process_video_input (e.g. for parallel audio\n extraction) pass it here to avoid re-downloading / re-decoding.\n\nReturns:\n List of paths to extracted frame images", "summary": "Process video input and extract frames.", "implementation": "Method `MLXMultimodalLM._prepare_video` calls `process_video_input`, `extract_video_frames_smart`, `save_frames_to_temp`; returns `save_frames_to_temp(frames)`.", "documented": true, "public": false, "addressable": true, "line": 1427, "end_line": 1463, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1427-L1463", "decorators": [], "calls": [ "process_video_input", "extract_video_frames_smart", "save_frames_to_temp" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "save_frames_to_temp(frames)" ], "awaits": false, "yields": false }, { "name": "_collect_video_inputs", "qualname": "MLXMultimodalLM._collect_video_inputs", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._collect_video_inputs", "kind": "method", "signature": "def _collect_video_inputs(self, messages: list[dict]) -> dict[int, list]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[int, list]", "docstring": "Collect video inputs from messages, keyed by message index.\n\nHandles both 'video' and 'video_url' content types, including\nPydantic model conversion.", "summary": "Collect video inputs from messages, keyed by message index.", "implementation": "Method `MLXMultimodalLM._collect_video_inputs` calls `enumerate`, `msg.get`, `isinstance`, `hasattr`; returns `video_inputs`.", "documented": true, "public": false, "addressable": true, "line": 1465, "end_line": 1497, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1465-L1497", "decorators": [], "calls": [ "enumerate", "msg.get", "isinstance", "hasattr", "item.model_dump", "item.dict().items", "item.dict", "item.get", "video_inputs.setdefault(msg_idx, []).append", "video_inputs.setdefault", "vid_url.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "video_inputs" ], "awaits": false, "yields": false }, { "name": "_collect_audio_inputs", "qualname": "MLXMultimodalLM._collect_audio_inputs", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._collect_audio_inputs", "kind": "method", "signature": "def _collect_audio_inputs(self, messages: list[dict]) -> dict[int, list]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[int, list]", "docstring": "Collect audio inputs from messages, keyed by message index.", "summary": "Collect audio inputs from messages, keyed by message index.", "implementation": "Method `MLXMultimodalLM._collect_audio_inputs` calls `enumerate`, `msg.get`, `isinstance`, `hasattr`; returns `audio_inputs`.", "documented": true, "public": false, "addressable": true, "line": 1499, "end_line": 1528, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1499-L1528", "decorators": [], "calls": [ "enumerate", "msg.get", "isinstance", "hasattr", "item.model_dump", "item.dict().items", "item.dict", "item.get", "audio_inputs.setdefault(msg_idx, []).append", "audio_inputs.setdefault", "audio_url.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "audio_inputs" ], "awaits": false, "yields": false }, { "name": "_prepare_native_video_inputs", "qualname": "MLXMultimodalLM._prepare_native_video_inputs", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._prepare_native_video_inputs", "kind": "method", "signature": "def _prepare_native_video_inputs(self, messages: list[dict], video_fps: float=DEFAULT_FPS, video_max_frames: int=MAX_FRAMES, tools: list | None=None) -> tuple[str, dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "Optional positional or keyword input; defaults to `DEFAULT_FPS`." }, { "name": "video_max_frames", "kind": "positional or keyword", "annotation": "int", "default": "MAX_FRAMES", "required": false, "description": "Optional positional or keyword input; defaults to `MAX_FRAMES`." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "tuple[str, dict]", "docstring": "Preprocess messages into prompt + generation kwargs for native video.\n\nMirrors the preprocessing in mlx_vlm.video_generate.main() so that\nupstream improvements are easy to adopt. Returns the formatted prompt\ntext and a dict of kwargs ready to pass to video_generate.generate().\n\nCurrently Qwen-family-specific (video_token_id / video_token_index).", "summary": "Preprocess messages into prompt + generation kwargs for native video.", "implementation": "Method `MLXMultimodalLM._prepare_native_video_inputs` calls `ImportError`, `self._translate_messages_for_native_video`, `self.processor.apply_chat_template`, `process_vision_info`; can raise `ImportError`; returns `(text, gen_kwargs)`.", "documented": true, "public": false, "addressable": true, "line": 1530, "end_line": 1648, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1530-L1648", "decorators": [], "calls": [ "ImportError", "self._translate_messages_for_native_video", "self.processor.apply_chat_template", "process_vision_info", "nmsg.get", "isinstance", "nitem.get", "audio_inputs.append", "self.processor", "mx.array", "inputs.get", "logger.info", "len", "gen_kwargs.get", "grid_thw_info.tolist" ], "state_reads": [ "self._translate_messages_for_native_video", "self.processor.apply_chat_template", "self.processor" ], "state_writes": [], "raises": [ "ImportError" ], "return_expressions": [ "(text, gen_kwargs)" ], "awaits": false, "yields": false }, { "name": "_generate_native_video", "qualname": "MLXMultimodalLM._generate_native_video", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._generate_native_video", "kind": "method", "signature": "def _generate_native_video(self, messages: list[dict], max_tokens: int=256, temperature: float=0.7, video_fps: float=DEFAULT_FPS, video_max_frames: int=MAX_FRAMES, tools: list | None=None, **kwargs) -> MLLMOutput", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Optional positional or keyword input; defaults to `256`." }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Optional positional or keyword input; defaults to `0.7`." }, { "name": "video_fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "Optional positional or keyword input; defaults to `DEFAULT_FPS`." }, { "name": "video_max_frames", "kind": "positional or keyword", "annotation": "int", "default": "MAX_FRAMES", "required": false, "description": "Optional positional or keyword input; defaults to `MAX_FRAMES`." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "MLLMOutput", "docstring": "Generate using native video pipeline (Qwen-family models).\n\nDelegates preprocessing to _prepare_native_video_inputs and generation\nto mlx_vlm.video_generate.generate(), keeping our code aligned with\nupstream's video pipeline so improvements are easy to adopt.", "summary": "Generate using native video pipeline (Qwen-family models).", "implementation": "Method `MLXMultimodalLM._generate_native_video` calls `ImportError`, `self._prepare_native_video_inputs`, `generate`, `hasattr`; can raise `ImportError`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1650, "end_line": 1695, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1650-L1695", "decorators": [], "calls": [ "ImportError", "self._prepare_native_video_inputs", "generate", "hasattr", "MLLMOutput", "getattr", "str" ], "state_reads": [ "self._prepare_native_video_inputs", "self.model", "self.processor" ], "state_writes": [], "raises": [ "ImportError" ], "return_expressions": [ "MLLMOutput(text=result.text, finish_reason='stop', prompt_tokens=getattr(result, 'prompt_tokens', 0), completion_tokens…", "MLLMOutput(text=str(result), finish_reason='stop')" ], "awaits": false, "yields": false }, { "name": "_translate_messages_for_native_video", "qualname": "MLXMultimodalLM._translate_messages_for_native_video", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM._translate_messages_for_native_video", "kind": "method", "signature": "def _translate_messages_for_native_video(self, messages: list[dict], video_fps: float, video_max_frames: int) -> list[dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_fps", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "video_max_frames", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Translate OpenAI API format messages to process_vision_info format.\n\nConverts video_url/video types and resolves remote/base64 inputs to local paths.\nImages are preserved as-is (process_vision_info handles them).", "summary": "Translate OpenAI API format messages to process_vision_info format.", "implementation": "Method `MLXMultimodalLM._translate_messages_for_native_video` calls `msg.get`, `isinstance`, `translated.append`, `str`; returns `translated`.", "documented": true, "public": false, "addressable": true, "line": 1697, "end_line": 1832, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1697-L1832", "decorators": [], "calls": [ "msg.get", "isinstance", "translated.append", "str", "hasattr", "item.model_dump", "item.dict().items", "item.dict", "probe.get", "new_content.append", "item.get", "img_url.get", "process_image_input", "vid_url.get", "process_video_input", "getattr", "extract_audio_from_video", "aud_url.get", "process_audio_input" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "translated" ], "awaits": false, "yields": false }, { "name": "generate", "qualname": "MLXMultimodalLM.generate", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.generate", "kind": "method", "signature": "def generate(self, prompt: str, images: list | None=None, videos: list | None=None, audio: list[str] | None=None, max_tokens: int=256, temperature: float=0.7, top_p: float=0.9, video_fps: float=DEFAULT_FPS, video_max_frames: int=MAX_FRAMES, use_cache: bool=True, **kwargs) -> MLLMOutput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt/question" }, { "name": "images", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "List of image URLs or base64 strings" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "List of video inputs (URLs, base64, or OpenAI format dicts)" }, { "name": "audio", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "List of audio file paths" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Top-p sampling parameter" }, { "name": "video_fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "FPS for video frame extraction (default: 2.0)" }, { "name": "video_max_frames", "kind": "positional or keyword", "annotation": "int", "default": "MAX_FRAMES", "required": false, "description": "Max frames to extract from video" }, { "name": "use_cache", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether to use KV cache (default: True)" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional generation parameters" } ], "return_annotation": "MLLMOutput", "docstring": "Generate text from multimodal input.\n\nArgs:\n prompt: Text prompt/question\n images: List of image URLs or base64 strings\n videos: List of video inputs (URLs, base64, or OpenAI format dicts)\n audio: List of audio file paths\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n top_p: Top-p sampling parameter\n video_fps: FPS for video frame extraction (default: 2.0)\n video_max_frames: Max frames to extract from video\n use_cache: Whether to use KV cache (default: True)\n **kwargs: Additional generation parameters\n\nReturns:\n MLLMOutput with generated text\n\nExample:\n # With local video\n output = model.generate(\"Describe this video\", videos=[\"video.mp4\"])\n\n # With video URL\n output = model.generate(\"What happens?\", videos=[\"https://example.com/video.mp4\"])\n\n # With base64 video\n output = model.generate(\"Describe\", videos=[\"data:video/mp4;base64,AAAA...\"])", "summary": "Generate text from multimodal input.", "implementation": "Method `MLXMultimodalLM.generate` calls `self.load`, `all_images.extend`, `self._prepare_images`, `all_sources.extend`; returns `MLLMOutput(text=output_text, finish_reason='stop', prompt_tokens=prompt_tokens, completion_tokens=generation_tokens, **…`.", "documented": true, "public": true, "addressable": true, "line": 1834, "end_line": 2002, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L1834-L2002", "decorators": [], "calls": [ "self.load", "all_images.extend", "self._prepare_images", "all_sources.extend", "self._prepare_video", "isinstance", "str", "all_sources.append", "logger.info", "len", "all_audio.extend", "self._prepare_audio", "hasattr", "apply_chat_template", "self._cache_manager.fetch_cache", "vlm_cache.make_prompt_cache", "self._reset_draft_metrics", "generate", "self._draft_generation_kwargs", "self._draft_metrics_since", "getattr", "self._cache_manager.store_cache", "logger.debug", "MLLMOutput" ], "state_reads": [ "self._loaded", "self.load", "self._prepare_images", "self._prepare_video", "self._prepare_audio", "self.processor", "self.config", "self._cache_manager", "self._cache_manager.fetch_cache", "self.model", "self.model.language_model", "self.max_kv_size", "self._reset_draft_metrics", "self._draft_generation_kwargs", "self._draft_metrics_since", "self._cache_manager.store_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "MLLMOutput(text=output_text, finish_reason='stop', prompt_tokens=prompt_tokens, completion_tokens=generation_tokens, **…" ], "awaits": false, "yields": false }, { "name": "stream_generate", "qualname": "MLXMultimodalLM.stream_generate", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.stream_generate", "kind": "method", "signature": "def stream_generate(self, prompt: str, images: list | None=None, videos: list[str] | None=None, audio: list[str] | None=None, max_tokens: int=256, temperature: float=0.7, video_fps: float=DEFAULT_FPS, **kwargs) -> Iterator[str]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt" }, { "name": "images", "kind": "positional or keyword", "annotation": "list | None", "default": "None", "required": false, "description": "List of image inputs" }, { "name": "videos", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "List of video paths" }, { "name": "audio", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "List of audio inputs" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "video_fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "FPS for video frame extraction" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters" } ], "return_annotation": "Iterator[str]", "docstring": "Stream text generation for multimodal input.\n\nArgs:\n prompt: Text prompt\n images: List of image inputs\n videos: List of video paths\n audio: List of audio inputs\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n video_fps: FPS for video frame extraction\n **kwargs: Additional parameters\n\nYields:\n Generated text chunks", "summary": "Stream text generation for multimodal input.", "implementation": "Method `MLXMultimodalLM.stream_generate` calls `self.load`, `self.generate`, `all_images.extend`, `self._prepare_images`; yields values incrementally; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 2004, "end_line": 2093, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2004-L2093", "decorators": [], "calls": [ "self.load", "self.generate", "all_images.extend", "self._prepare_images", "self._prepare_video", "all_audio.extend", "self._prepare_audio", "apply_chat_template", "len", "stream_generate", "self._draft_generation_kwargs" ], "state_reads": [ "self._loaded", "self.load", "self.generate", "self._prepare_images", "self._prepare_video", "self._prepare_audio", "self.processor", "self.config", "self.model", "self._draft_generation_kwargs" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": true }, { "name": "chat", "qualname": "MLXMultimodalLM.chat", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.chat", "kind": "method", "signature": "def chat(self, messages: list[dict], max_tokens: int=256, temperature: float=0.7, **kwargs) -> MLLMOutput", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "List of chat messages (OpenAI format)" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters" } ], "return_annotation": "MLLMOutput", "docstring": "Chat with OpenAI-compatible message format.\n\nSupports multimodal content in messages:\n- {\"type\": \"text\", \"text\": \"...\"}\n- {\"type\": \"image_url\", \"image_url\": {\"url\": \"...\"}}\n- {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/...;base64,...\"}}\n\nArgs:\n messages: List of chat messages (OpenAI format)\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n **kwargs: Additional parameters\n\nReturns:\n MLLMOutput with assistant's response", "summary": "Chat with OpenAI-compatible message format.", "implementation": "Method `MLXMultimodalLM.chat` calls `self.load`, `logger.info`, `len`, `kwargs.pop`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 2095, "end_line": 2487, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2095-L2487", "decorators": [], "calls": [ "self.load", "logger.info", "len", "kwargs.pop", "chat_template_kwargs.pop", "self._collect_video_inputs", "self._collect_audio_inputs", "self._generate_native_video", "_msg_video_inputs.items", "bool", "_msg_audio_inputs.get", "process_video_input", "logger.warning", "extract_audio_from_video", "_msg_extra_audio.setdefault(msg_idx, []).append", "_msg_extra_audio.setdefault", "self._prepare_video", "all_video_frames.extend", "_msg_extra_audio.items", "_msg_audio_inputs.setdefault(msg_idx, []).extend", "_msg_audio_inputs.setdefault", "_msg_audio_inputs.values", "all_audio_inputs.extend", "_build_mllm_chat_messages", "all_images.extend", "self._prepare_images", "self._prepare_audio", "enumerate", "str", "cm.get", "template_extra_kwargs.update", "get_chat_template", "template_extra_kwargs.pop", "reversed", "m.get", "isinstance", "item.get", "hasattr", "tokenizer.encode", "self._cache_manager.fetch", "time.time", "copy.copy", "mx.array", "prompt_cache.append", "vlm_cache.make_prompt_cache", "self._reset_draft_metrics", "generate", "self._draft_generation_kwargs", "self._draft_metrics_since", "getattr", "min", "cache_to_store.append", "self._cache_manager.store", "MLLMOutput" ], "state_reads": [ "self._loaded", "self.load", "self._collect_video_inputs", "self._collect_audio_inputs", "self._video_native", "self._generate_native_video", "self._video_native_with_audio", "self._prepare_video", "self._prepare_images", "self._prepare_audio", "self.processor", "self.processor.tokenizer", "self._cache_manager", "self._cache_manager.fetch", "self.model", "self.model.language_model", "self.max_kv_size", "self._reset_draft_metrics", "self._draft_generation_kwargs", "self._draft_metrics_since", "self._cache_manager.store", "self.model_name" ], "state_writes": [], "raises": [], "return_expressions": [ "self._generate_native_video(messages=messages, max_tokens=max_tokens, temperature=temperature, video_fps=video_fps, vid…", "MLLMOutput(text=output_text, finish_reason='stop', prompt_tokens=prompt_tokens, completion_tokens=generation_tokens, **…" ], "awaits": false, "yields": false }, { "name": "stream_chat", "qualname": "MLXMultimodalLM.stream_chat", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.stream_chat", "kind": "method", "signature": "def stream_chat(self, messages: list[dict], max_tokens: int=256, temperature: float=0.7, **kwargs) -> Iterator[MLLMOutput]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "List of chat messages (OpenAI format)" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens to generate" }, { "name": "temperature", "kind": "positional or keyword", "annotation": "float", "default": "0.7", "required": false, "description": "Sampling temperature" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters" } ], "return_annotation": "Iterator[MLLMOutput]", "docstring": "Stream chat with OpenAI-compatible message format.\n\nSupports multimodal content in messages:\n- {\"type\": \"text\", \"text\": \"...\"}\n- {\"type\": \"image_url\", \"image_url\": {\"url\": \"...\"}}\n- {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/...;base64,...\"}}\n\nArgs:\n messages: List of chat messages (OpenAI format)\n max_tokens: Maximum tokens to generate\n temperature: Sampling temperature\n **kwargs: Additional parameters\n\nYields:\n MLLMOutput with incremental text chunks", "summary": "Stream chat with OpenAI-compatible message format.", "implementation": "Method `MLXMultimodalLM.stream_chat` calls `self.load`, `self.chat`, `kwargs.pop`, `chat_template_kwargs.pop`; yields values incrementally; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 2489, "end_line": 2737, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2489-L2737", "decorators": [], "calls": [ "self.load", "self.chat", "kwargs.pop", "chat_template_kwargs.pop", "self._collect_video_inputs", "self._collect_audio_inputs", "self._generate_native_video", "_msg_video_inputs.items", "bool", "_msg_audio_inputs.get", "process_video_input", "logger.warning", "extract_audio_from_video", "_msg_extra_audio.setdefault(msg_idx, []).append", "_msg_extra_audio.setdefault", "self._prepare_video", "all_video_frames.extend", "len", "logger.info", "_msg_extra_audio.items", "_msg_audio_inputs.setdefault(msg_idx, []).extend", "_msg_audio_inputs.setdefault", "_msg_audio_inputs.values", "all_audio_inputs.extend", "_build_mllm_chat_messages", "all_images.extend", "self._prepare_images", "self._prepare_audio", "template_extra_kwargs.update", "get_chat_template", "template_extra_kwargs.pop", "reversed", "m.get", "isinstance", "item.get", "self._cache_manager.fetch_cache", "logger.debug", "vlm_cache.make_prompt_cache", "self._reset_draft_metrics", "stream_generate", "self._draft_generation_kwargs", "hasattr", "str", "MLLMOutput", "getattr", "dir", "self._draft_metrics_since" ], "state_reads": [ "self._loaded", "self.load", "self.chat", "self._collect_video_inputs", "self._collect_audio_inputs", "self._video_native", "self._generate_native_video", "self._video_native_with_audio", "self._prepare_video", "self._prepare_images", "self._prepare_audio", "self.processor", "self._cache_manager", "self._cache_manager.fetch_cache", "self.model", "self.model.language_model", "self.max_kv_size", "self._reset_draft_metrics", "self._draft_generation_kwargs", "self._draft_metrics_since" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": true }, { "name": "describe_image", "qualname": "MLXMultimodalLM.describe_image", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.describe_image", "kind": "method", "signature": "def describe_image(self, image: str, prompt: str='Describe this image in detail.', max_tokens: int=512, **kwargs) -> str", "parameters": [ { "name": "image", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Image path, URL, or base64 string" }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "'Describe this image in detail.'", "required": false, "description": "Description prompt" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "512", "required": false, "description": "Maximum tokens" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters" } ], "return_annotation": "str", "docstring": "Convenience method to describe an image.\n\nArgs:\n image: Image path, URL, or base64 string\n prompt: Description prompt\n max_tokens: Maximum tokens\n **kwargs: Additional parameters\n\nReturns:\n Image description text", "summary": "Convenience method to describe an image.", "implementation": "Method `MLXMultimodalLM.describe_image` calls `self.generate`; returns `output.text`.", "documented": true, "public": true, "addressable": true, "line": 2739, "end_line": 2764, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2739-L2764", "decorators": [], "calls": [ "self.generate" ], "state_reads": [ "self.generate" ], "state_writes": [], "raises": [], "return_expressions": [ "output.text" ], "awaits": false, "yields": false }, { "name": "answer_about_image", "qualname": "MLXMultimodalLM.answer_about_image", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.answer_about_image", "kind": "method", "signature": "def answer_about_image(self, image: str, question: str, max_tokens: int=256, **kwargs) -> str", "parameters": [ { "name": "image", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Image path, URL, or base64 string" }, { "name": "question", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Question about the image" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "256", "required": false, "description": "Maximum tokens" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters" } ], "return_annotation": "str", "docstring": "Answer a question about an image.\n\nArgs:\n image: Image path, URL, or base64 string\n question: Question about the image\n max_tokens: Maximum tokens\n **kwargs: Additional parameters\n\nReturns:\n Answer text", "summary": "Answer a question about an image.", "implementation": "Method `MLXMultimodalLM.answer_about_image` calls `self.generate`; returns `output.text`.", "documented": true, "public": true, "addressable": true, "line": 2766, "end_line": 2791, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2766-L2791", "decorators": [], "calls": [ "self.generate" ], "state_reads": [ "self.generate" ], "state_writes": [], "raises": [], "return_expressions": [ "output.text" ], "awaits": false, "yields": false }, { "name": "describe_video", "qualname": "MLXMultimodalLM.describe_video", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.describe_video", "kind": "method", "signature": "def describe_video(self, video: str | dict, prompt: str='Describe what happens in this video.', fps: float=2.0, max_frames: int=32, max_tokens: int=512, **kwargs) -> str", "parameters": [ { "name": "video", "kind": "positional or keyword", "annotation": "str | dict", "default": "", "required": true, "description": "Video file path, URL, base64, or OpenAI format dict" }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "'Describe what happens in this video.'", "required": false, "description": "Description prompt" }, { "name": "fps", "kind": "positional or keyword", "annotation": "float", "default": "2.0", "required": false, "description": "Frames per second to extract" }, { "name": "max_frames", "kind": "positional or keyword", "annotation": "int", "default": "32", "required": false, "description": "Maximum frames to extract" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "512", "required": false, "description": "Maximum tokens to generate" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "str", "docstring": "Describe a video using frame extraction.\n\nArgs:\n video: Video file path, URL, base64, or OpenAI format dict\n prompt: Description prompt\n fps: Frames per second to extract\n max_frames: Maximum frames to extract\n max_tokens: Maximum tokens to generate\n\nReturns:\n Video description text\n\nExample:\n # URL\n model.describe_video(\"https://example.com/video.mp4\")\n\n # OpenAI format\n model.describe_video({\"url\": \"https://example.com/video.mp4\"})", "summary": "Describe a video using frame extraction.", "implementation": "Method `MLXMultimodalLM.describe_video` calls `self.generate`; returns `output.text`.", "documented": true, "public": true, "addressable": true, "line": 2793, "end_line": 2830, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2793-L2830", "decorators": [], "calls": [ "self.generate" ], "state_reads": [ "self.generate" ], "state_writes": [], "raises": [], "return_expressions": [ "output.text" ], "awaits": false, "yields": false }, { "name": "get_cache_stats", "qualname": "MLXMultimodalLM.get_cache_stats", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.get_cache_stats", "kind": "method", "signature": "def get_cache_stats(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get MLLM cache statistics.\n\nReturns:\n Dictionary with cache stats (hits, misses, hit_rate, tokens_saved, etc.)", "summary": "Get MLLM cache statistics.", "implementation": "Method `MLXMultimodalLM.get_cache_stats` calls `self._cache_manager.get_stats`, `len`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 2832, "end_line": 2846, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2832-L2846", "decorators": [], "calls": [ "self._cache_manager.get_stats", "len" ], "state_reads": [ "self._cache_manager", "self._cache_manager.get_stats", "self._cache_manager.max_size" ], "state_writes": [], "raises": [], "return_expressions": [ "{'enabled': False}", "stats" ], "awaits": false, "yields": false }, { "name": "clear_cache", "qualname": "MLXMultimodalLM.clear_cache", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.clear_cache", "kind": "method", "signature": "def clear_cache(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear the MLLM KV cache.", "summary": "Clear the MLLM KV cache.", "implementation": "Method `MLXMultimodalLM.clear_cache` calls `self._cache_manager.clear`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 2848, "end_line": 2852, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2848-L2852", "decorators": [], "calls": [ "self._cache_manager.clear", "logger.info" ], "state_reads": [ "self._cache_manager", "self._cache_manager.clear" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_model_info", "qualname": "MLXMultimodalLM.get_model_info", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.get_model_info", "kind": "method", "signature": "def get_model_info(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get information about the loaded model.", "summary": "Get information about the loaded model.", "implementation": "Method `MLXMultimodalLM.get_model_info` calls `getattr`, `self._cache_manager.get_stats`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 2854, "end_line": 2874, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2854-L2874", "decorators": [], "calls": [ "getattr", "self._cache_manager.get_stats" ], "state_reads": [ "self._loaded", "self.model_name", "self.enable_cache", "self.config", "self._cache_manager", "self._cache_manager.get_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "{'loaded': False, 'model_name': self.model_name}", "info" ], "awaits": false, "yields": false }, { "name": "list_supported_model_families", "qualname": "MLXMultimodalLM.list_supported_model_families", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.list_supported_model_families", "kind": "method", "signature": "def list_supported_model_families() -> dict[str, str]", "parameters": [], "return_annotation": "dict[str, str]", "docstring": "List supported model families and their patterns.\n\nAny model on HuggingFace containing these patterns in the name\nis likely compatible with mlx-vlm.", "summary": "List supported model families and their patterns.", "implementation": "Method `MLXMultimodalLM.list_supported_model_families` returns `{'Qwen-VL': 'Qwen VL models (Qwen2-VL, Qwen2.5-VL, Qwen3-VL, etc.)', 'LLaVA': 'LLaVA vision-language models', 'Idefics'…`.", "documented": true, "public": true, "addressable": true, "line": 2877, "end_line": 2897, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2877-L2897", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'Qwen-VL': 'Qwen VL models (Qwen2-VL, Qwen2.5-VL, Qwen3-VL, etc.)', 'LLaVA': 'LLaVA vision-language models', 'Idefics'…" ], "awaits": false, "yields": false }, { "name": "is_mllm_model", "qualname": "MLXMultimodalLM.is_mllm_model", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.is_mllm_model", "kind": "method", "signature": "def is_mllm_model(model_name: str) -> bool", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if a model name indicates an MLLM model.", "summary": "Check if a model name indicates an MLLM model.", "implementation": "Method `MLXMultimodalLM.is_mllm_model` calls `model_name.lower`, `any`, `pattern.lower`; returns `any((pattern.lower() in model_lower for pattern in mllm_patterns))`.", "documented": true, "public": true, "addressable": true, "line": 2900, "end_line": 2934, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2900-L2934", "decorators": [ "staticmethod" ], "calls": [ "model_name.lower", "any", "pattern.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "any((pattern.lower() in model_lower for pattern in mllm_patterns))" ], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "MLXMultimodalLM.__repr__", "full_name": "vllm_mlx.models.mllm.MLXMultimodalLM.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `MLXMultimodalLM.__repr__` returns `f''`.", "implementation": "Method `MLXMultimodalLM.__repr__` returns `f''`.", "documented": false, "public": false, "addressable": true, "line": 2936, "end_line": 2938, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/models/mllm.py#L2936-L2938", "decorators": [], "calls": [], "state_reads": [ "self._loaded", "self.model_name" ], "state_writes": [], "raises": [], "return_expressions": [ "f''" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.multimodal_processor", "path": "vllm_mlx/multimodal_processor.py", "page_path": "reference/api/vllm_mlx/multimodal_processor.md", "docstring": "Multimodal processor for VLM continuous batching.\n\nThis module handles preprocessing of multimodal inputs (images, videos)\nfor use with the continuous batching scheduler. It extracts processed\ninputs that can be batched together efficiently.", "summary": "Multimodal processor for VLM continuous batching.", "line_count": 431, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L1-L431", "members": [ "logger", "ProcessedMultimodalInput", "MultimodalProcessor" ], "symbols": [ { "name": "ProcessedMultimodalInput", "qualname": "ProcessedMultimodalInput", "full_name": "vllm_mlx.multimodal_processor.ProcessedMultimodalInput", "kind": "class", "signature": "class ProcessedMultimodalInput", "parameters": [ { "name": "input_ids", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "pixel_values", "kind": "field", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "attention_mask", "kind": "field", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "image_grid_thw", "kind": "field", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "num_images", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "num_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "extra_kwargs", "kind": "field", "annotation": "Dict[str, Any]", "default": "field(default_factory=dict)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=dict)`." } ], "return_annotation": "ProcessedMultimodalInput", "docstring": "Container for processed multimodal inputs ready for batching.\n\nAttributes:\n input_ids: Tokenized text with image/video tokens (mx.array)\n pixel_values: Processed image tensors (mx.array)\n attention_mask: Attention mask for the input (mx.array)\n image_grid_thw: Grid info for Qwen-VL models (mx.array)\n num_images: Number of images in this input\n num_tokens: Number of tokens in input_ids\n extra_kwargs: Additional model-specific kwargs", "summary": "Container for processed multimodal inputs ready for batching.", "implementation": "Class `ProcessedMultimodalInput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 29, "end_line": 49, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L29-L49", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "MultimodalProcessor", "qualname": "MultimodalProcessor", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor", "kind": "class", "signature": "class MultimodalProcessor", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM model (for config access)" }, { "name": "processor", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM processor (tokenizer + image processor)" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional model config" } ], "return_annotation": "MultimodalProcessor", "docstring": "Processor for preparing multimodal inputs for VLM batching.\n\nThis class wraps mlx_vlm's prepare_inputs function and provides\na clean interface for the scheduler to preprocess requests.\n\nExample:\n >>> processor = MultimodalProcessor(model, vlm_processor)\n >>> processed = processor.process(\n ... prompt=\"What's in this image?\",\n ... images=[\"photo.jpg\"]\n ... )\n >>> # processed.input_ids, processed.pixel_values ready for batching", "summary": "Processor for preparing multimodal inputs for VLM batching.", "implementation": "Class `MultimodalProcessor` declares 8 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 52, "end_line": 431, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L52-L431", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MultimodalProcessor.__init__", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.__init__", "kind": "method", "signature": "def __init__(self, model: Any, processor: Any, config: Optional[Any]=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM model (for config access)" }, { "name": "processor", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The VLM processor (tokenizer + image processor)" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional model config" } ], "return_annotation": "not annotated", "docstring": "Initialize the multimodal processor.\n\nArgs:\n model: The VLM model (for config access)\n processor: The VLM processor (tokenizer + image processor)\n config: Optional model config", "summary": "Initialize the multimodal processor.", "implementation": "Method `MultimodalProcessor.__init__` updates `self.model`, `self.processor`, `self.config`, `self.tokenizer`; calls `getattr`, `hasattr`.", "documented": true, "public": false, "addressable": true, "line": 68, "end_line": 94, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L68-L94", "decorators": [], "calls": [ "getattr", "hasattr" ], "state_reads": [ "self.config" ], "state_writes": [ "self.model", "self.processor", "self.config", "self.tokenizer", "self.image_token_index" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "process", "qualname": "MultimodalProcessor.process", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.process", "kind": "method", "signature": "def process(self, prompt: str, images: Optional[List[str]]=None, videos: Optional[List[str]]=None, video_fps: float=DEFAULT_FPS, video_max_frames: int=MAX_FRAMES, add_special_tokens: bool=True, **kwargs) -> ProcessedMultimodalInput", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt (already formatted with chat template)" }, { "name": "images", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of image URLs or base64 strings" }, { "name": "videos", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of video URLs or base64 inputs" }, { "name": "video_fps", "kind": "positional or keyword", "annotation": "float", "default": "DEFAULT_FPS", "required": false, "description": "FPS for video frame extraction" }, { "name": "video_max_frames", "kind": "positional or keyword", "annotation": "int", "default": "MAX_FRAMES", "required": false, "description": "Max frames per video" }, { "name": "add_special_tokens", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether to add special tokens" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional model-specific parameters" } ], "return_annotation": "ProcessedMultimodalInput", "docstring": "Process multimodal inputs for batching.\n\nArgs:\n prompt: Text prompt (already formatted with chat template)\n images: List of image URLs or base64 strings\n videos: List of video URLs or base64 inputs\n video_fps: FPS for video frame extraction\n video_max_frames: Max frames per video\n add_special_tokens: Whether to add special tokens\n **kwargs: Additional model-specific parameters\n\nReturns:\n ProcessedMultimodalInput with all processed tensors", "summary": "Process multimodal inputs for batching.", "implementation": "Method `MultimodalProcessor.process` calls `process_image_input`, `all_images.append`, `logger.warning`, `process_video_input`; returns `ProcessedMultimodalInput(input_ids=input_ids, pixel_values=pixel_values, attention_mask=attention_mask, image_grid_thw=…`.", "documented": true, "public": true, "addressable": true, "line": 96, "end_line": 186, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L96-L186", "decorators": [], "calls": [ "process_image_input", "all_images.append", "logger.warning", "process_video_input", "extract_video_frames_smart", "save_frames_to_temp", "all_images.extend", "logger.debug", "len", "hasattr", "prepare_inputs", "inputs.get", "inputs.items", "extra_kwargs.pop", "ProcessedMultimodalInput" ], "state_reads": [ "self.config", "self.config.model_type", "self.processor", "self.image_token_index" ], "state_writes": [], "raises": [], "return_expressions": [ "ProcessedMultimodalInput(input_ids=input_ids, pixel_values=pixel_values, attention_mask=attention_mask, image_grid_thw=…" ], "awaits": false, "yields": false }, { "name": "process_for_request", "qualname": "MultimodalProcessor.process_for_request", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.process_for_request", "kind": "method", "signature": "def process_for_request(self, prompt: str, images: Optional[List[str]]=None, videos: Optional[List[str]]=None, **kwargs) -> Dict[str, Any]", "parameters": [ { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text prompt" }, { "name": "images", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of image inputs" }, { "name": "videos", "kind": "positional or keyword", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "List of video inputs" }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional parameters" } ], "return_annotation": "Dict[str, Any]", "docstring": "Process inputs and return a dict suitable for Request fields.\n\nThis is a convenience method that returns the processed data\nin a format that can be directly assigned to Request fields.\n\nArgs:\n prompt: Text prompt\n images: List of image inputs\n videos: List of video inputs\n **kwargs: Additional parameters\n\nReturns:\n Dict with keys matching Request multimodal fields", "summary": "Process inputs and return a dict suitable for Request fields.", "implementation": "Method `MultimodalProcessor.process_for_request` calls `self.process`, `processed.input_ids.tolist`; returns `{'prompt_token_ids': processed.input_ids.tolist() if processed.input_ids is not None else None, 'num_prompt_tokens': pr…`.", "documented": true, "public": true, "addressable": true, "line": 188, "end_line": 224, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L188-L224", "decorators": [], "calls": [ "self.process", "processed.input_ids.tolist" ], "state_reads": [ "self.process" ], "state_writes": [], "raises": [], "return_expressions": [ "{'prompt_token_ids': processed.input_ids.tolist() if processed.input_ids is not None else None, 'num_prompt_tokens': pr…" ], "awaits": false, "yields": false }, { "name": "batch_pixel_values", "qualname": "MultimodalProcessor.batch_pixel_values", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.batch_pixel_values", "kind": "method", "signature": "def batch_pixel_values(self, pixel_values_list: List[Optional[mx.array]]) -> Optional[mx.array]", "parameters": [ { "name": "pixel_values_list", "kind": "positional or keyword", "annotation": "List[Optional[mx.array]]", "default": "", "required": true, "description": "List of pixel_values from multiple requests" } ], "return_annotation": "Optional[mx.array]", "docstring": "Batch multiple pixel_values tensors together.\n\nFor VLM batching, we need to concatenate pixel values from\nmultiple requests. This handles the case where some requests\nmay not have images.\n\nArgs:\n pixel_values_list: List of pixel_values from multiple requests\n\nReturns:\n Batched pixel_values or None if no images", "summary": "Batch multiple pixel_values tensors together.", "implementation": "Method `MultimodalProcessor.batch_pixel_values` calls `mx.concatenate`, `logger.warning`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 226, "end_line": 255, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L226-L255", "decorators": [], "calls": [ "mx.concatenate", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "mx.concatenate(valid_pixels, axis=0)", "valid_pixels[0] if valid_pixels else None" ], "awaits": false, "yields": false }, { "name": "batch_image_grid_thw", "qualname": "MultimodalProcessor.batch_image_grid_thw", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.batch_image_grid_thw", "kind": "method", "signature": "def batch_image_grid_thw(self, grid_thw_list: List[Optional[mx.array]]) -> Optional[mx.array]", "parameters": [ { "name": "grid_thw_list", "kind": "positional or keyword", "annotation": "List[Optional[mx.array]]", "default": "", "required": true, "description": "List of image_grid_thw from multiple requests" } ], "return_annotation": "Optional[mx.array]", "docstring": "Batch multiple image_grid_thw tensors together.\n\nArgs:\n grid_thw_list: List of image_grid_thw from multiple requests\n\nReturns:\n Batched image_grid_thw or None", "summary": "Batch multiple image_grid_thw tensors together.", "implementation": "Method `MultimodalProcessor.batch_image_grid_thw` calls `mx.concatenate`, `logger.warning`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 257, "end_line": 279, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L257-L279", "decorators": [], "calls": [ "mx.concatenate", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "mx.concatenate(valid_grids, axis=0)", "valid_grids[0] if valid_grids else None" ], "awaits": false, "yields": false }, { "name": "prepare_for_batch", "qualname": "MultimodalProcessor.prepare_for_batch", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.prepare_for_batch", "kind": "method", "signature": "def prepare_for_batch(self, processed_inputs: List[ProcessedMultimodalInput]) -> Tuple[mx.array, Dict[str, Any], List[int]]", "parameters": [ { "name": "processed_inputs", "kind": "positional or keyword", "annotation": "List[ProcessedMultimodalInput]", "default": "", "required": true, "description": "List of ProcessedMultimodalInput from process()" } ], "return_annotation": "Tuple[mx.array, Dict[str, Any], List[int]]", "docstring": "Prepare multiple processed inputs for batch generation.\n\nThis method takes a list of ProcessedMultimodalInput objects and\ncombines them into batched tensors suitable for the MLLMBatchGenerator.\n\nArgs:\n processed_inputs: List of ProcessedMultimodalInput from process()\n\nReturns:\n Tuple of:\n - input_ids: Left-padded input tokens [batch_size, max_seq_len]\n - batch_kwargs: Dict with batched pixel_values, attention_mask, etc.\n - padding_amounts: List of padding amounts for each request", "summary": "Prepare multiple processed inputs for batch generation.", "implementation": "Method `MultimodalProcessor.prepare_for_batch` calls `mx.array`, `max`, `zip`, `padded_ids.append`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 281, "end_line": 366, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L281-L366", "decorators": [], "calls": [ "mx.array", "max", "zip", "padded_ids.append", "hasattr", "ids.tolist", "list", "self.batch_pixel_values", "self.batch_image_grid_thw", "padded_masks.append", "mx.ones", "mask.reshape", "mx.zeros", "mx.concatenate", "mx.stack", "logger.warning", "merged_extra.update", "batch_kwargs.items" ], "state_reads": [ "self.batch_pixel_values", "self.batch_image_grid_thw" ], "state_writes": [], "raises": [], "return_expressions": [ "(mx.array([]), {}, [])", "(input_ids, batch_kwargs, padding_amounts)" ], "awaits": false, "yields": false }, { "name": "extract_vision_embeddings", "qualname": "MultimodalProcessor.extract_vision_embeddings", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.extract_vision_embeddings", "kind": "method", "signature": "def extract_vision_embeddings(self, pixel_values: mx.array, image_grid_thw: Optional[mx.array]=None) -> mx.array", "parameters": [ { "name": "pixel_values", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Processed image tensors" }, { "name": "image_grid_thw", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional grid info for Qwen-VL models" } ], "return_annotation": "mx.array", "docstring": "Extract vision embeddings from pixel values.\n\nThis runs the vision encoder part of the VLM to get embeddings\nthat can be cached and reused.\n\nArgs:\n pixel_values: Processed image tensors\n image_grid_thw: Optional grid info for Qwen-VL models\n\nReturns:\n Vision embeddings tensor", "summary": "Extract vision embeddings from pixel values.", "implementation": "Method `MultimodalProcessor.extract_vision_embeddings` calls `hasattr`, `ValueError`, `getattr`, `vision_encoder`; can raise `ValueError`; returns `embeddings`.", "documented": true, "public": true, "addressable": true, "line": 368, "end_line": 409, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L368-L409", "decorators": [], "calls": [ "hasattr", "ValueError", "getattr", "vision_encoder" ], "state_reads": [ "self.model" ], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "embeddings" ], "awaits": false, "yields": false }, { "name": "compute_vision_hash", "qualname": "MultimodalProcessor.compute_vision_hash", "full_name": "vllm_mlx.multimodal_processor.MultimodalProcessor.compute_vision_hash", "kind": "method", "signature": "def compute_vision_hash(self, pixel_values: mx.array) -> str", "parameters": [ { "name": "pixel_values", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Processed image tensors" } ], "return_annotation": "str", "docstring": "Compute a hash for pixel values for caching purposes.\n\nArgs:\n pixel_values: Processed image tensors\n\nReturns:\n Hash string for the vision inputs", "summary": "Compute a hash for pixel values for caching purposes.", "implementation": "Method `MultimodalProcessor.compute_vision_hash` calls `str`, `pixel_values.reshape(-1)[:100].tolist`, `pixel_values.reshape`, `hashlib.sha256(hash_input.encode()).hexdigest`; returns `hashlib.sha256(hash_input.encode()).hexdigest()[:16]`.", "documented": true, "public": true, "addressable": true, "line": 411, "end_line": 431, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/multimodal_processor.py#L411-L431", "decorators": [], "calls": [ "str", "pixel_values.reshape(-1)[:100].tolist", "pixel_values.reshape", "hashlib.sha256(hash_input.encode()).hexdigest", "hashlib.sha256", "hash_input.encode" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hashlib.sha256(hash_input.encode()).hexdigest()[:16]" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.optimizations", "path": "vllm_mlx/optimizations.py", "page_path": "reference/api/vllm_mlx/optimizations.md", "docstring": "Hardware detection and system information for vllm-mlx.\n\nThis module provides:\n- Hardware detection for Apple Silicon (M1, M2, M3, M4 series)\n- System memory detection\n- Memory bandwidth benchmarking\n\nNote: mlx-lm already includes optimized implementations internally:\n- Flash Attention via mx.fast.scaled_dot_product_attention\n- Efficient memory management\n- Optimized Metal kernels\n\nNo additional optimization is needed - mlx-lm is already fast out of the box.\n\nUsage:\n from vllm_mlx.optimizations import (\n detect_hardware,\n get_optimization_status,\n benchmark_memory_bandwidth,\n )", "summary": "Hardware detection and system information for vllm-mlx.", "line_count": 209, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/optimizations.py#L1-L209", "members": [ "logger", "HardwareInfo", "HARDWARE_PROFILES", "get_system_memory_gb", "detect_hardware", "benchmark_memory_bandwidth", "get_optimization_status" ], "symbols": [ { "name": "HardwareInfo", "qualname": "HardwareInfo", "full_name": "vllm_mlx.optimizations.HardwareInfo", "kind": "class", "signature": "class HardwareInfo", "parameters": [ { "name": "chip_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "total_memory_gb", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "memory_bandwidth_gbs", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." }, { "name": "gpu_cores", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "HardwareInfo", "docstring": "Hardware information for Apple Silicon.", "summary": "Hardware information for Apple Silicon.", "implementation": "Class `HardwareInfo` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 40, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/optimizations.py#L34-L40", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_system_memory_gb", "qualname": "get_system_memory_gb", "full_name": "vllm_mlx.optimizations.get_system_memory_gb", "kind": "function", "signature": "def get_system_memory_gb() -> float", "parameters": [], "return_annotation": "float", "docstring": "Get actual system memory in GB.\n\nReturns:\n Total system memory in GB (unified memory on Apple Silicon)", "summary": "Get actual system memory in GB.", "implementation": "Function `get_system_memory_gb` calls `subprocess.run`, `int`, `result.stdout.strip`, `mx.device_info`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 68, "end_line": 94, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/optimizations.py#L68-L94", "decorators": [], "calls": [ "subprocess.run", "int", "result.stdout.strip", "mx.device_info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mem_bytes / 1024 ** 3", "device_info['memory_size'] / 1024 ** 3", "16.0" ], "awaits": false, "yields": false }, { "name": "detect_hardware", "qualname": "detect_hardware", "full_name": "vllm_mlx.optimizations.detect_hardware", "kind": "function", "signature": "def detect_hardware() -> HardwareInfo", "parameters": [], "return_annotation": "HardwareInfo", "docstring": "Detect Apple Silicon hardware and return info.\n\nMemory is detected dynamically from the system.\nOther specs (bandwidth, GPU cores) come from known chip profiles.\n\nReturns:\n HardwareInfo with detected hardware specifications", "summary": "Detect Apple Silicon hardware and return info.", "implementation": "Function `detect_hardware` calls `mx.device_info`, `device_info.get`, `get_system_memory_gb`, `sorted`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 97, "end_line": 141, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/optimizations.py#L97-L141", "decorators": [], "calls": [ "mx.device_info", "device_info.get", "get_system_memory_gb", "sorted", "HARDWARE_PROFILES.items", "HardwareInfo", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "HardwareInfo(chip_name=chip_name, total_memory_gb=actual_memory_gb, memory_bandwidth_gbs=profile['bandwidth'], gpu_core…", "HardwareInfo(chip_name='Unknown', total_memory_gb=actual_memory_gb, memory_bandwidth_gbs=200, gpu_cores=16)", "HardwareInfo(chip_name='Unknown', total_memory_gb=get_system_memory_gb(), memory_bandwidth_gbs=200, gpu_cores=16)" ], "awaits": false, "yields": false }, { "name": "benchmark_memory_bandwidth", "qualname": "benchmark_memory_bandwidth", "full_name": "vllm_mlx.optimizations.benchmark_memory_bandwidth", "kind": "function", "signature": "def benchmark_memory_bandwidth() -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Benchmark actual memory bandwidth achieved.\n\nReturns:\n dict with bandwidth measurements for different array sizes", "summary": "Benchmark actual memory bandwidth achieved.", "implementation": "Function `benchmark_memory_bandwidth` calls `mx.random.normal`, `mx.eval`, `time.perf_counter`, `range`; returns `results`.", "documented": true, "public": true, "addressable": true, "line": 144, "end_line": 174, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/optimizations.py#L144-L174", "decorators": [], "calls": [ "mx.random.normal", "mx.eval", "time.perf_counter", "range" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "get_optimization_status", "qualname": "get_optimization_status", "full_name": "vllm_mlx.optimizations.get_optimization_status", "kind": "function", "signature": "def get_optimization_status() -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get current hardware and MLX status.\n\nReturns:\n dict with hardware info and MLX configuration", "summary": "Get current hardware and MLX status.", "implementation": "Function `get_optimization_status` calls `detect_hardware`, `mx.device_info`, `hasattr`, `device_info.get`; returns `{'hardware': {'chip': hw.chip_name, 'total_memory_gb': hw.total_memory_gb, 'memory_bandwidth_gbs': hw.memory_bandwidth_…`.", "documented": true, "public": true, "addressable": true, "line": 177, "end_line": 209, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/optimizations.py#L177-L209", "decorators": [], "calls": [ "detect_hardware", "mx.device_info", "hasattr", "device_info.get", "mx.get_active_memory", "mx.get_cache_memory", "mx.get_peak_memory" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'hardware': {'chip': hw.chip_name, 'total_memory_gb': hw.total_memory_gb, 'memory_bandwidth_gbs': hw.memory_bandwidth_…" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.output_collector", "path": "vllm_mlx/output_collector.py", "page_path": "reference/api/vllm_mlx/output_collector.md", "docstring": "Output collector for streaming with low-latency optimizations.\n\nThis module implements the RequestOutputCollector pattern from vLLM,\nproviding non-blocking output collection with intelligent aggregation.", "summary": "Output collector for streaming with low-latency optimizations.", "line_count": 212, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L1-L212", "members": [ "RequestOutputCollector", "RequestStreamState" ], "symbols": [ { "name": "RequestOutputCollector", "qualname": "RequestOutputCollector", "full_name": "vllm_mlx.output_collector.RequestOutputCollector", "kind": "class", "signature": "class RequestOutputCollector", "parameters": [ { "name": "aggregate", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, merge outputs when producer gets ahead. This prevents buffer explosion under load." } ], "return_annotation": "RequestOutputCollector", "docstring": "Per-request output collector with smart buffering.\n\nThis class implements the vLLM pattern for efficient streaming:\n- Non-blocking get_nowait() to avoid unnecessary task switches\n- Output aggregation when producer is faster than consumer\n- Event-based signaling for efficient waiting\n- Tracking of active consumers for yield optimization\n\nUsage:\n collector = RequestOutputCollector()\n\n # Producer side (engine loop)\n collector.put(output)\n\n # Consumer side (streaming generator)\n output = collector.get_nowait() or await collector.get()", "summary": "Per-request output collector with smart buffering.", "implementation": "Class `RequestOutputCollector` declares 7 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 17, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L17-L170", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "RequestOutputCollector.__init__", "full_name": "vllm_mlx.output_collector.RequestOutputCollector.__init__", "kind": "method", "signature": "def __init__(self, aggregate: bool=True)", "parameters": [ { "name": "aggregate", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "If True, merge outputs when producer gets ahead. This prevents buffer explosion under load." } ], "return_annotation": "not annotated", "docstring": "Initialize the collector.\n\nArgs:\n aggregate: If True, merge outputs when producer gets ahead.\n This prevents buffer explosion under load.", "summary": "Initialize the collector.", "implementation": "Method `RequestOutputCollector.__init__` updates `self.output`, `self.ready`, `self.aggregate`, `self._is_waiting`; calls `asyncio.Event`.", "documented": true, "public": false, "addressable": true, "line": 42, "end_line": 53, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L42-L53", "decorators": [], "calls": [ "asyncio.Event" ], "state_reads": [], "state_writes": [ "self.output", "self.ready", "self.aggregate", "self._is_waiting" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "put", "qualname": "RequestOutputCollector.put", "full_name": "vllm_mlx.output_collector.RequestOutputCollector.put", "kind": "method", "signature": "def put(self, output: RequestOutput) -> None", "parameters": [ { "name": "output", "kind": "positional or keyword", "annotation": "RequestOutput", "default": "", "required": true, "description": "The RequestOutput to store" } ], "return_annotation": "None", "docstring": "Put an output into the collector (non-blocking).\n\nIf aggregation is enabled and an output already exists,\nthe new output is merged with the existing one.\n\nArgs:\n output: The RequestOutput to store", "summary": "Put an output into the collector (non-blocking).", "implementation": "Method `RequestOutputCollector.put` updates `self.output`; calls `self._merge_outputs`, `self.ready.set`.", "documented": true, "public": true, "addressable": true, "line": 55, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L55-L73", "decorators": [], "calls": [ "self._merge_outputs", "self.ready.set" ], "state_reads": [ "self.output", "self.aggregate", "self._merge_outputs", "self.ready.set", "self.ready" ], "state_writes": [ "self.output" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_nowait", "qualname": "RequestOutputCollector.get_nowait", "full_name": "vllm_mlx.output_collector.RequestOutputCollector.get_nowait", "kind": "method", "signature": "def get_nowait(self) -> Optional[RequestOutput]", "parameters": [], "return_annotation": "Optional[RequestOutput]", "docstring": "Get output without blocking.\n\nThis avoids task switching when output is available,\nreducing latency under load.\n\nReturns:\n The output if available, None otherwise", "summary": "Get output without blocking.", "implementation": "Method `RequestOutputCollector.get_nowait` updates `self.output`; calls `self.ready.clear`; returns `output`.", "documented": true, "public": true, "addressable": true, "line": 75, "end_line": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L75-L89", "decorators": [], "calls": [ "self.ready.clear" ], "state_reads": [ "self.output", "self.ready.clear", "self.ready" ], "state_writes": [ "self.output" ], "raises": [], "return_expressions": [ "output" ], "awaits": false, "yields": false }, { "name": "get", "qualname": "RequestOutputCollector.get", "full_name": "vllm_mlx.output_collector.RequestOutputCollector.get", "kind": "method", "signature": "async def get(self) -> RequestOutput", "parameters": [], "return_annotation": "RequestOutput", "docstring": "Get output, blocking only if none available.\n\nThis method blocks until an output is available.\nFor low-latency streaming, prefer:\n output = collector.get_nowait() or await collector.get()\n\nReturns:\n The RequestOutput", "summary": "Get output, blocking only if none available.", "implementation": "Method `RequestOutputCollector.get` updates `self._is_waiting`; calls `self.ready.wait`, `self.get_nowait`; awaits asynchronous work; returns `output`.", "documented": true, "public": true, "addressable": true, "line": 91, "end_line": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L91-L118", "decorators": [], "calls": [ "self.ready.wait", "self.get_nowait" ], "state_reads": [ "self._is_waiting", "self.output", "self.ready.wait", "self.ready", "self.get_nowait" ], "state_writes": [ "self._is_waiting" ], "raises": [], "return_expressions": [ "output" ], "awaits": true, "yields": false }, { "name": "_merge_outputs", "qualname": "RequestOutputCollector._merge_outputs", "full_name": "vllm_mlx.output_collector.RequestOutputCollector._merge_outputs", "kind": "method", "signature": "def _merge_outputs(self, existing: RequestOutput, new: RequestOutput) -> RequestOutput", "parameters": [ { "name": "existing", "kind": "positional or keyword", "annotation": "RequestOutput", "default": "", "required": true, "description": "The existing output in the buffer" }, { "name": "new", "kind": "positional or keyword", "annotation": "RequestOutput", "default": "", "required": true, "description": "The new output to merge" } ], "return_annotation": "RequestOutput", "docstring": "Merge two outputs when producer gets ahead of consumer.\n\nThis combines the token lists and text, keeping the latest\nstatus information.\n\nArgs:\n existing: The existing output in the buffer\n new: The new output to merge\n\nReturns:\n Merged RequestOutput", "summary": "Merge two outputs when producer gets ahead of consumer.", "implementation": "Method `RequestOutputCollector._merge_outputs` calls `RequestOutput`; returns `RequestOutput(request_id=new.request_id, new_token_ids=merged_new_token_ids, new_text=merged_new_text, output_token_ids…`.", "documented": true, "public": false, "addressable": true, "line": 120, "end_line": 152, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L120-L152", "decorators": [], "calls": [ "RequestOutput" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "RequestOutput(request_id=new.request_id, new_token_ids=merged_new_token_ids, new_text=merged_new_text, output_token_ids…" ], "awaits": false, "yields": false }, { "name": "clear", "qualname": "RequestOutputCollector.clear", "full_name": "vllm_mlx.output_collector.RequestOutputCollector.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear any pending output.", "summary": "Clear any pending output.", "implementation": "Method `RequestOutputCollector.clear` updates `self.output`, `self._is_waiting`; calls `self.ready.clear`.", "documented": true, "public": true, "addressable": true, "line": 154, "end_line": 161, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L154-L161", "decorators": [], "calls": [ "self.ready.clear" ], "state_reads": [ "self.ready.clear", "self.ready", "self._is_waiting" ], "state_writes": [ "self.output", "self._is_waiting" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "has_waiting_consumers", "qualname": "RequestOutputCollector.has_waiting_consumers", "full_name": "vllm_mlx.output_collector.RequestOutputCollector.has_waiting_consumers", "kind": "method", "signature": "def has_waiting_consumers(cls) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if any collector has waiting consumers.\n\nUsed by engine to optimize: only yield when someone is waiting.", "summary": "Check if any collector has waiting consumers.", "implementation": "Method `RequestOutputCollector.has_waiting_consumers` returns `cls._waiting_consumers > 0`.", "documented": true, "public": true, "addressable": true, "line": 164, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L164-L170", "decorators": [ "classmethod" ], "calls": [], "state_reads": [ "cls._waiting_lock", "cls._waiting_consumers" ], "state_writes": [], "raises": [], "return_expressions": [ "cls._waiting_consumers > 0" ], "awaits": false, "yields": false }, { "name": "RequestStreamState", "qualname": "RequestStreamState", "full_name": "vllm_mlx.output_collector.RequestStreamState", "kind": "class", "signature": "class RequestStreamState", "parameters": [ { "name": "stream_interval", "kind": "field", "annotation": "int", "default": "1", "required": false, "description": "Optional constructor field; defaults to `1`." }, { "name": "sent_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "RequestStreamState", "docstring": "Tracks streaming state for a request.\n\nThis is used to implement stream_interval batching,\nallowing tokens to be accumulated before sending.", "summary": "Tracks streaming state for a request.", "implementation": "Class `RequestStreamState` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 174, "end_line": 212, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L174-L212", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "should_send", "qualname": "RequestStreamState.should_send", "full_name": "vllm_mlx.output_collector.RequestStreamState.should_send", "kind": "method", "signature": "def should_send(self, total_tokens: int, finished: bool) -> bool", "parameters": [ { "name": "total_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Total tokens generated so far" }, { "name": "finished", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Whether generation is complete" } ], "return_annotation": "bool", "docstring": "Determine if output should be sent based on stream_interval.\n\nArgs:\n total_tokens: Total tokens generated so far\n finished: Whether generation is complete\n\nReturns:\n True if output should be sent", "summary": "Determine if output should be sent based on stream_interval.", "implementation": "Method `RequestStreamState.should_send` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 185, "end_line": 203, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L185-L203", "decorators": [], "calls": [], "state_reads": [ "self.sent_tokens", "self.stream_interval" ], "state_writes": [], "raises": [], "return_expressions": [ "True", "total_tokens - self.sent_tokens >= self.stream_interval" ], "awaits": false, "yields": false }, { "name": "mark_sent", "qualname": "RequestStreamState.mark_sent", "full_name": "vllm_mlx.output_collector.RequestStreamState.mark_sent", "kind": "method", "signature": "def mark_sent(self, total_tokens: int) -> None", "parameters": [ { "name": "total_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Total tokens at time of send" } ], "return_annotation": "None", "docstring": "Update state after sending output.\n\nArgs:\n total_tokens: Total tokens at time of send", "summary": "Update state after sending output.", "implementation": "Method `RequestStreamState.mark_sent` updates `self.sent_tokens`.", "documented": true, "public": true, "addressable": true, "line": 205, "end_line": 212, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/output_collector.py#L205-L212", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.sent_tokens" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.paged_cache", "path": "vllm_mlx/paged_cache.py", "page_path": "reference/api/vllm_mlx/paged_cache.md", "docstring": "Paged KV Cache Manager for vllm-mlx.\n\nThis module implements block-based paged KV cache management following vLLM's\narchitecture (vllm/v1/core/block_pool.py), adapted for MLX on Apple Silicon.\n\nKey components:\n- KVCacheBlock: Metadata for each cache block with doubly linked list pointers\n- FreeKVCacheBlockQueue: O(1) doubly linked list for LRU block allocation\n- BlockHashToBlockMap: Hash-to-block cache for prefix caching\n- PagedCacheManager: Main manager with block allocation, prefix caching, and COW\n\nFeatures:\n- Block-based allocation (configurable tokens per block)\n- Reference counting for shared blocks\n- Copy-on-Write (COW) for efficient prefix sharing\n- LRU eviction using doubly linked list (O(1) operations)\n- Chain hashing for prefix caching (hash depends on parent block)\n\nReference: vLLM v1 - vllm/v1/core/block_pool.py, vllm/v1/core/kv_cache_utils.py", "summary": "Paged KV Cache Manager for vllm-mlx.", "line_count": 1197, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1-L1197", "members": [ "logger", "BlockHash", "compute_block_hash", "CacheBlock", "KVCacheBlock", "FreeKVCacheBlockQueue", "BlockHashToBlockMap", "BlockTable", "CacheStats", "PagedCacheManager" ], "symbols": [ { "name": "compute_block_hash", "qualname": "compute_block_hash", "full_name": "vllm_mlx.paged_cache.compute_block_hash", "kind": "function", "signature": "def compute_block_hash(parent_hash: Optional[BlockHash], token_ids: List[int], extra_keys: Optional[Tuple[Any, ...]]=None) -> BlockHash", "parameters": [ { "name": "parent_hash", "kind": "positional or keyword", "annotation": "Optional[BlockHash]", "default": "", "required": true, "description": "Hash of the previous block, or None for first block" }, { "name": "token_ids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Token IDs in this block" }, { "name": "extra_keys", "kind": "positional or keyword", "annotation": "Optional[Tuple[Any, ...]]", "default": "None", "required": false, "description": "Additional keys (e.g., LoRA, multimodal)" } ], "return_annotation": "BlockHash", "docstring": "Compute hash for a block based on its content and parent block.\n\nThis enables prefix caching by creating a chain of hashes where\neach block's hash depends on all previous blocks (similar to vLLM).\n\nArgs:\n parent_hash: Hash of the previous block, or None for first block\n token_ids: Token IDs in this block\n extra_keys: Additional keys (e.g., LoRA, multimodal)\n\nReturns:\n Content-based hash for this block", "summary": "Compute hash for a block based on its content and parent block.", "implementation": "Function `compute_block_hash` calls `hashlib.sha256`, `hasher.update`, `bytes`, `str`; returns `BlockHash(hasher.digest())`.", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 75, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L40-L75", "decorators": [], "calls": [ "hashlib.sha256", "hasher.update", "bytes", "str", "tuple", "BlockHash", "hasher.digest" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "BlockHash(hasher.digest())" ], "awaits": false, "yields": false }, { "name": "CacheBlock", "qualname": "CacheBlock", "full_name": "vllm_mlx.paged_cache.CacheBlock", "kind": "class", "signature": "class CacheBlock", "parameters": [ { "name": "block_id", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "ref_count", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "block_hash", "kind": "field", "annotation": "Optional[BlockHash]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prev_free_block", "kind": "field", "annotation": "Optional['CacheBlock']", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "next_free_block", "kind": "field", "annotation": "Optional['CacheBlock']", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "is_null", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "cache_data", "kind": "field", "annotation": "Optional[List[Tuple[Any, Any]]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "token_count", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "hash_value", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "last_access", "kind": "field", "annotation": "float", "default": "field(default_factory=time.time)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=time.time)`." } ], "return_annotation": "CacheBlock", "docstring": "KV cache block metadata following vLLM's design.\n\nEach block represents a fixed number of tokens (block_size) worth\nof KV cache data. Blocks can be shared across requests via\nreference counting for prefix caching.\n\nAttributes:\n block_id: Physical block index (0 to num_blocks - 1)\n ref_count: Reference count for sharing (0 = can be evicted)\n block_hash: Content hash for prefix caching (None if not cached)\n prev_free_block: Previous block in free list (doubly linked)\n next_free_block: Next block in free list (doubly linked)\n is_null: True if this is the null/placeholder block\n cache_data: Actual KV tensor data stored in this block\n token_count: Number of tokens stored in this block", "summary": "KV cache block metadata following vLLM's design.", "implementation": "Class `CacheBlock` declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 84, "end_line": 146, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L84-L146", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_full", "qualname": "CacheBlock.is_full", "full_name": "vllm_mlx.paged_cache.CacheBlock.is_full", "kind": "method", "signature": "def is_full(self, block_size: int) -> bool", "parameters": [ { "name": "block_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if block is at capacity.", "summary": "Check if block is at capacity.", "implementation": "Method `CacheBlock.is_full` returns `self.token_count >= block_size`.", "documented": true, "public": true, "addressable": true, "line": 123, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L123-L125", "decorators": [], "calls": [], "state_reads": [ "self.token_count" ], "state_writes": [], "raises": [], "return_expressions": [ "self.token_count >= block_size" ], "awaits": false, "yields": false }, { "name": "is_shared", "qualname": "CacheBlock.is_shared", "full_name": "vllm_mlx.paged_cache.CacheBlock.is_shared", "kind": "method", "signature": "def is_shared(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if block is shared (ref_count > 1).", "summary": "Check if block is shared (ref_count > 1).", "implementation": "Method `CacheBlock.is_shared` returns `self.ref_count > 1`.", "documented": true, "public": true, "addressable": true, "line": 127, "end_line": 129, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L127-L129", "decorators": [], "calls": [], "state_reads": [ "self.ref_count" ], "state_writes": [], "raises": [], "return_expressions": [ "self.ref_count > 1" ], "awaits": false, "yields": false }, { "name": "reset_hash", "qualname": "CacheBlock.reset_hash", "full_name": "vllm_mlx.paged_cache.CacheBlock.reset_hash", "kind": "method", "signature": "def reset_hash(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset block hash when evicted from cache.", "summary": "Reset block hash when evicted from cache.", "implementation": "Method `CacheBlock.reset_hash` updates `self.block_hash`, `self.hash_value`.", "documented": true, "public": true, "addressable": true, "line": 131, "end_line": 134, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L131-L134", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.block_hash", "self.hash_value" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "touch", "qualname": "CacheBlock.touch", "full_name": "vllm_mlx.paged_cache.CacheBlock.touch", "kind": "method", "signature": "def touch(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Update last access time.", "summary": "Update last access time.", "implementation": "Method `CacheBlock.touch` updates `self.last_access`; calls `time.time`.", "documented": true, "public": true, "addressable": true, "line": 136, "end_line": 138, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L136-L138", "decorators": [], "calls": [ "time.time" ], "state_reads": [], "state_writes": [ "self.last_access" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "CacheBlock.__repr__", "full_name": "vllm_mlx.paged_cache.CacheBlock.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `CacheBlock.__repr__` returns `f'CacheBlock(id={self.block_id}, ref={self.ref_count}, tokens={self.token_count}, prev={prev_id}, next={next_id})'`.", "implementation": "Method `CacheBlock.__repr__` returns `f'CacheBlock(id={self.block_id}, ref={self.ref_count}, tokens={self.token_count}, prev={prev_id}, next={next_id})'`.", "documented": false, "public": false, "addressable": true, "line": 140, "end_line": 146, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L140-L146", "decorators": [], "calls": [], "state_reads": [ "self.prev_free_block", "self.prev_free_block.block_id", "self.next_free_block", "self.next_free_block.block_id", "self.block_id", "self.ref_count", "self.token_count" ], "state_writes": [], "raises": [], "return_expressions": [ "f'CacheBlock(id={self.block_id}, ref={self.ref_count}, tokens={self.token_count}, prev={prev_id}, next={next_id})'" ], "awaits": false, "yields": false }, { "name": "FreeKVCacheBlockQueue", "qualname": "FreeKVCacheBlockQueue", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue", "kind": "class", "signature": "class FreeKVCacheBlockQueue", "parameters": [ { "name": "blocks", "kind": "positional or keyword", "annotation": "List[CacheBlock]", "default": "", "required": true, "description": "List of all CacheBlock objects" } ], "return_annotation": "FreeKVCacheBlockQueue", "docstring": "Doubly linked list of free blocks following vLLM's design.\n\nProvides O(1) operations for:\n- popleft(): Allocate block from front (LRU order)\n- remove(): Remove block from middle (when touched by cache hit)\n- append(): Return block to end (when freed)\n\nThe queue maintains LRU eviction order:\n- Front = least recently used (evict first)\n- Back = most recently used (evict last)\n\nUses fake head/tail sentinels to simplify edge cases.", "summary": "Doubly linked list of free blocks following vLLM's design.", "implementation": "Class `FreeKVCacheBlockQueue` declares 7 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 158, "end_line": 337, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L158-L337", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "FreeKVCacheBlockQueue.__init__", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue.__init__", "kind": "method", "signature": "def __init__(self, blocks: List[CacheBlock]) -> None", "parameters": [ { "name": "blocks", "kind": "positional or keyword", "annotation": "List[CacheBlock]", "default": "", "required": true, "description": "List of all CacheBlock objects" } ], "return_annotation": "None", "docstring": "Initialize queue with all blocks as free.\n\nArgs:\n blocks: List of all CacheBlock objects", "summary": "Initialize queue with all blocks as free.", "implementation": "Method `FreeKVCacheBlockQueue.__init__` updates `self.num_free_blocks`, `self.fake_head`, `self.fake_tail`, `self.fake_head.next_free_block`; calls `len`, `range`, `CacheBlock`.", "documented": true, "public": false, "addressable": true, "line": 174, "end_line": 201, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L174-L201", "decorators": [], "calls": [ "len", "range", "CacheBlock" ], "state_reads": [ "self.fake_head", "self.fake_tail" ], "state_writes": [ "self.num_free_blocks", "self.fake_head", "self.fake_tail", "self.fake_head.next_free_block", "self.fake_tail.prev_free_block" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "popleft", "qualname": "FreeKVCacheBlockQueue.popleft", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue.popleft", "kind": "method", "signature": "def popleft(self) -> CacheBlock", "parameters": [], "return_annotation": "CacheBlock", "docstring": "Pop and return the first (LRU) free block.\n\nRaises:\n ValueError: If no free blocks available", "summary": "Pop and return the first (LRU) free block.", "implementation": "Method `FreeKVCacheBlockQueue.popleft` updates `self.fake_head.next_free_block`, `self.num_free_blocks`; calls `ValueError`; can raise `ValueError`; returns `block`.", "documented": true, "public": true, "addressable": true, "line": 203, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L203-L225", "decorators": [], "calls": [ "ValueError" ], "state_reads": [ "self.fake_head.next_free_block", "self.fake_head", "self.fake_tail" ], "state_writes": [ "self.fake_head.next_free_block", "self.num_free_blocks" ], "raises": [ "ValueError" ], "return_expressions": [ "block" ], "awaits": false, "yields": false }, { "name": "popleft_n", "qualname": "FreeKVCacheBlockQueue.popleft_n", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue.popleft_n", "kind": "method", "signature": "def popleft_n(self, n: int) -> List[CacheBlock]", "parameters": [ { "name": "n", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of blocks to allocate" } ], "return_annotation": "List[CacheBlock]", "docstring": "Pop n blocks from the front.\n\nArgs:\n n: Number of blocks to allocate\n\nReturns:\n List of n free blocks\n\nRaises:\n AssertionError: If not enough free blocks", "summary": "Pop n blocks from the front.", "implementation": "Method `FreeKVCacheBlockQueue.popleft_n` updates `self.fake_head.next_free_block`, `self.num_free_blocks`; calls `range`, `result.append`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 227, "end_line": 265, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L227-L265", "decorators": [], "calls": [ "range", "result.append" ], "state_reads": [ "self.num_free_blocks", "self.fake_head.next_free_block", "self.fake_head", "self.fake_tail" ], "state_writes": [ "self.fake_head.next_free_block", "self.num_free_blocks" ], "raises": [], "return_expressions": [ "[]", "result" ], "awaits": false, "yields": false }, { "name": "remove", "qualname": "FreeKVCacheBlockQueue.remove", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue.remove", "kind": "method", "signature": "def remove(self, block: CacheBlock) -> None", "parameters": [ { "name": "block", "kind": "positional or keyword", "annotation": "CacheBlock", "default": "", "required": true, "description": "Block to remove" } ], "return_annotation": "None", "docstring": "Remove a block from the middle of the queue.\n\nUsed when a free block is \"touched\" (reused by prefix cache hit).\n\nArgs:\n block: Block to remove\n\nRaises:\n RuntimeError: If block not in queue", "summary": "Remove a block from the middle of the queue.", "implementation": "Method `FreeKVCacheBlockQueue.remove` updates `self.num_free_blocks`; calls `RuntimeError`; can raise `RuntimeError`.", "documented": true, "public": true, "addressable": true, "line": 267, "end_line": 288, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L267-L288", "decorators": [], "calls": [ "RuntimeError" ], "state_reads": [], "state_writes": [ "self.num_free_blocks" ], "raises": [ "RuntimeError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "append", "qualname": "FreeKVCacheBlockQueue.append", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue.append", "kind": "method", "signature": "def append(self, block: CacheBlock) -> None", "parameters": [ { "name": "block", "kind": "positional or keyword", "annotation": "CacheBlock", "default": "", "required": true, "description": "Block to append" } ], "return_annotation": "None", "docstring": "Append a block to the end (MRU position).\n\nArgs:\n block: Block to append", "summary": "Append a block to the end (MRU position).", "implementation": "Method `FreeKVCacheBlockQueue.append` updates `self.fake_tail.prev_free_block`, `self.num_free_blocks`.", "documented": true, "public": true, "addressable": true, "line": 290, "end_line": 305, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L290-L305", "decorators": [], "calls": [], "state_reads": [ "self.fake_tail.prev_free_block", "self.fake_tail" ], "state_writes": [ "self.fake_tail.prev_free_block", "self.num_free_blocks" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "append_n", "qualname": "FreeKVCacheBlockQueue.append_n", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue.append_n", "kind": "method", "signature": "def append_n(self, blocks: List[CacheBlock]) -> None", "parameters": [ { "name": "blocks", "kind": "positional or keyword", "annotation": "List[CacheBlock]", "default": "", "required": true, "description": "Blocks to append (in order)" } ], "return_annotation": "None", "docstring": "Append multiple blocks to the end.\n\nArgs:\n blocks: Blocks to append (in order)", "summary": "Append multiple blocks to the end.", "implementation": "Method `FreeKVCacheBlockQueue.append_n` updates `self.fake_tail.prev_free_block`, `self.num_free_blocks`; calls `len`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 307, "end_line": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L307-L328", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.fake_tail.prev_free_block", "self.fake_tail" ], "state_writes": [ "self.fake_tail.prev_free_block", "self.num_free_blocks" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_all_free_blocks", "qualname": "FreeKVCacheBlockQueue.get_all_free_blocks", "full_name": "vllm_mlx.paged_cache.FreeKVCacheBlockQueue.get_all_free_blocks", "kind": "method", "signature": "def get_all_free_blocks(self) -> List[CacheBlock]", "parameters": [], "return_annotation": "List[CacheBlock]", "docstring": "Get all free blocks (for testing).", "summary": "Get all free blocks (for testing).", "implementation": "Method `FreeKVCacheBlockQueue.get_all_free_blocks` calls `result.append`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 330, "end_line": 337, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L330-L337", "decorators": [], "calls": [ "result.append" ], "state_reads": [ "self.fake_head.next_free_block", "self.fake_head", "self.fake_tail" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "BlockHashToBlockMap", "qualname": "BlockHashToBlockMap", "full_name": "vllm_mlx.paged_cache.BlockHashToBlockMap", "kind": "class", "signature": "class BlockHashToBlockMap", "parameters": [], "return_annotation": "BlockHashToBlockMap", "docstring": "Cache mapping block hashes to blocks for prefix caching.\n\nFollows vLLM's design where the same hash can map to multiple\nblocks (for different KV cache groups in hybrid models).", "summary": "Cache mapping block hashes to blocks for prefix caching.", "implementation": "Class `BlockHashToBlockMap` declares 6 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 345, "end_line": 407, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L345-L407", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "BlockHashToBlockMap.__init__", "full_name": "vllm_mlx.paged_cache.BlockHashToBlockMap.__init__", "kind": "method", "signature": "def __init__(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `BlockHashToBlockMap.__init__` updates `self._cache`.", "implementation": "Method `BlockHashToBlockMap.__init__` updates `self._cache`.", "documented": false, "public": false, "addressable": true, "line": 353, "end_line": 354, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L353-L354", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._cache" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_block", "qualname": "BlockHashToBlockMap.get_block", "full_name": "vllm_mlx.paged_cache.BlockHashToBlockMap.get_block", "kind": "method", "signature": "def get_block(self, block_hash: BlockHash) -> Optional[CacheBlock]", "parameters": [ { "name": "block_hash", "kind": "positional or keyword", "annotation": "BlockHash", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[CacheBlock]", "docstring": "Get any block with the given hash.", "summary": "Get any block with the given hash.", "implementation": "Method `BlockHashToBlockMap.get_block` calls `self._cache.get`, `isinstance`, `next`, `iter`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 356, "end_line": 365, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L356-L365", "decorators": [], "calls": [ "self._cache.get", "isinstance", "next", "iter", "blocks.values" ], "state_reads": [ "self._cache.get", "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "blocks", "next(iter(blocks.values()))" ], "awaits": false, "yields": false }, { "name": "insert", "qualname": "BlockHashToBlockMap.insert", "full_name": "vllm_mlx.paged_cache.BlockHashToBlockMap.insert", "kind": "method", "signature": "def insert(self, block_hash: BlockHash, block: CacheBlock) -> None", "parameters": [ { "name": "block_hash", "kind": "positional or keyword", "annotation": "BlockHash", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "block", "kind": "positional or keyword", "annotation": "CacheBlock", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Insert a block into the cache.", "summary": "Insert a block into the cache.", "implementation": "Method `BlockHashToBlockMap.insert` calls `self._cache.get`, `isinstance`.", "documented": true, "public": true, "addressable": true, "line": 367, "end_line": 378, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L367-L378", "decorators": [], "calls": [ "self._cache.get", "isinstance" ], "state_reads": [ "self._cache.get", "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "pop", "qualname": "BlockHashToBlockMap.pop", "full_name": "vllm_mlx.paged_cache.BlockHashToBlockMap.pop", "kind": "method", "signature": "def pop(self, block_hash: BlockHash, block_id: int) -> Optional[CacheBlock]", "parameters": [ { "name": "block_hash", "kind": "positional or keyword", "annotation": "BlockHash", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "block_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[CacheBlock]", "docstring": "Remove and return a specific block from the cache.", "summary": "Remove and return a specific block from the cache.", "implementation": "Method `BlockHashToBlockMap.pop` calls `self._cache.pop`, `isinstance`, `blocks.pop`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 380, "end_line": 399, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L380-L399", "decorators": [], "calls": [ "self._cache.pop", "isinstance", "blocks.pop" ], "state_reads": [ "self._cache.pop", "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "blocks", "block" ], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "BlockHashToBlockMap.__len__", "full_name": "vllm_mlx.paged_cache.BlockHashToBlockMap.__len__", "kind": "method", "signature": "def __len__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "", "summary": "Method `BlockHashToBlockMap.__len__` calls `len`; returns `len(self._cache)`.", "implementation": "Method `BlockHashToBlockMap.__len__` calls `len`; returns `len(self._cache)`.", "documented": false, "public": false, "addressable": true, "line": 401, "end_line": 402, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L401-L402", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self._cache)" ], "awaits": false, "yields": false }, { "name": "clear", "qualname": "BlockHashToBlockMap.clear", "full_name": "vllm_mlx.paged_cache.BlockHashToBlockMap.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Remove every block-hash mapping without mutating the blocks.", "summary": "Remove every block-hash mapping without mutating the blocks.", "implementation": "Method `BlockHashToBlockMap.clear` calls `self._cache.clear`.", "documented": true, "public": true, "addressable": true, "line": 404, "end_line": 407, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L404-L407", "decorators": [], "calls": [ "self._cache.clear" ], "state_reads": [ "self._cache.clear", "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "BlockTable", "qualname": "BlockTable", "full_name": "vllm_mlx.paged_cache.BlockTable", "kind": "class", "signature": "class BlockTable", "parameters": [ { "name": "request_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "block_ids", "kind": "field", "annotation": "List[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "num_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "BlockTable", "docstring": "Per-request block table mapping logical to physical blocks.\n\nSimilar to vLLM's block table, this maps a request's token positions\nto physical cache blocks.\n\nAttributes:\n request_id: Unique request identifier\n block_ids: List of physical block IDs\n num_tokens: Total number of cached tokens", "summary": "Per-request block table mapping logical to physical blocks.", "implementation": "Class `BlockTable` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 416, "end_line": 447, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L416-L447", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "add_block", "qualname": "BlockTable.add_block", "full_name": "vllm_mlx.paged_cache.BlockTable.add_block", "kind": "method", "signature": "def add_block(self, block_id: int, num_tokens: int) -> None", "parameters": [ { "name": "block_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Add a block to the table.", "summary": "Add a block to the table.", "implementation": "Method `BlockTable.add_block` updates `self.num_tokens`; calls `self.block_ids.append`.", "documented": true, "public": true, "addressable": true, "line": 433, "end_line": 436, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L433-L436", "decorators": [], "calls": [ "self.block_ids.append" ], "state_reads": [ "self.block_ids.append", "self.block_ids" ], "state_writes": [ "self.num_tokens" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "BlockTable.__len__", "full_name": "vllm_mlx.paged_cache.BlockTable.__len__", "kind": "method", "signature": "def __len__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "", "summary": "Method `BlockTable.__len__` calls `len`; returns `len(self.block_ids)`.", "implementation": "Method `BlockTable.__len__` calls `len`; returns `len(self.block_ids)`.", "documented": false, "public": false, "addressable": true, "line": 438, "end_line": 439, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L438-L439", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.block_ids" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.block_ids)" ], "awaits": false, "yields": false }, { "name": "copy", "qualname": "BlockTable.copy", "full_name": "vllm_mlx.paged_cache.BlockTable.copy", "kind": "method", "signature": "def copy(self, new_request_id: str) -> 'BlockTable'", "parameters": [ { "name": "new_request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "'BlockTable'", "docstring": "Create a copy with new request ID.", "summary": "Create a copy with new request ID.", "implementation": "Method `BlockTable.copy` calls `BlockTable`, `self.block_ids.copy`; returns `BlockTable(request_id=new_request_id, block_ids=self.block_ids.copy(), num_tokens=self.num_tokens)`.", "documented": true, "public": true, "addressable": true, "line": 441, "end_line": 447, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L441-L447", "decorators": [], "calls": [ "BlockTable", "self.block_ids.copy" ], "state_reads": [ "self.block_ids.copy", "self.block_ids", "self.num_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "BlockTable(request_id=new_request_id, block_ids=self.block_ids.copy(), num_tokens=self.num_tokens)" ], "awaits": false, "yields": false }, { "name": "CacheStats", "qualname": "CacheStats", "full_name": "vllm_mlx.paged_cache.CacheStats", "kind": "class", "signature": "class CacheStats", "parameters": [ { "name": "total_blocks", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "allocated_blocks", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "free_blocks", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "shared_blocks", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "total_tokens_cached", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "cache_hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "cache_misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "cow_copies", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "evictions", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "CacheStats", "docstring": "Statistics for cache monitoring.", "summary": "Statistics for cache monitoring.", "implementation": "Class `CacheStats` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 456, "end_line": 467, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L456-L467", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "PagedCacheManager", "qualname": "PagedCacheManager", "full_name": "vllm_mlx.paged_cache.PagedCacheManager", "kind": "class", "signature": "class PagedCacheManager", "parameters": [ { "name": "block_size", "kind": "positional or keyword", "annotation": "int", "default": "64", "required": false, "description": "Number of tokens per block (default: 64)" }, { "name": "max_blocks", "kind": "positional or keyword", "annotation": "int", "default": "1000", "required": false, "description": "Maximum number of blocks to allocate (default: 1000)" }, { "name": "enable_caching", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether to enable prefix caching (default: True)" } ], "return_annotation": "PagedCacheManager", "docstring": "Paged KV cache manager following vLLM's BlockPool architecture.\n\nFeatures:\n- Block allocation/deallocation with reference counting\n- Prefix sharing via chain-based hash deduplication\n- Copy-on-Write for efficient forking\n- O(1) LRU eviction using doubly linked list\n\nArgs:\n block_size: Number of tokens per block (default: 64)\n max_blocks: Maximum number of blocks to allocate (default: 1000)\n enable_caching: Whether to enable prefix caching (default: True)", "summary": "Paged KV cache manager following vLLM's BlockPool architecture.", "implementation": "Class `PagedCacheManager` declares 34 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 475, "end_line": 1197, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L475-L1197", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "PagedCacheManager.__init__", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.__init__", "kind": "method", "signature": "def __init__(self, block_size: int=64, max_blocks: int=1000, enable_caching: bool=True)", "parameters": [ { "name": "block_size", "kind": "positional or keyword", "annotation": "int", "default": "64", "required": false, "description": "Optional positional or keyword input; defaults to `64`." }, { "name": "max_blocks", "kind": "positional or keyword", "annotation": "int", "default": "1000", "required": false, "description": "Optional positional or keyword input; defaults to `1000`." }, { "name": "enable_caching", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Optional positional or keyword input; defaults to `True`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `PagedCacheManager.__init__` updates `self.block_size`, `self.max_blocks`, `self.enable_caching`, `self.blocks`; calls `CacheBlock`, `range`, `FreeKVCacheBlockQueue`, `BlockHashToBlockMap`.", "implementation": "Method `PagedCacheManager.__init__` updates `self.block_size`, `self.max_blocks`, `self.enable_caching`, `self.blocks`; calls `CacheBlock`, `range`, `FreeKVCacheBlockQueue`, `BlockHashToBlockMap`.", "documented": false, "public": false, "addressable": true, "line": 491, "end_line": 540, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L491-L540", "decorators": [], "calls": [ "CacheBlock", "range", "FreeKVCacheBlockQueue", "BlockHashToBlockMap", "self.free_block_queue.popleft", "CacheStats", "threading.RLock", "logger.info" ], "state_reads": [ "self.blocks", "self.free_block_queue.popleft", "self.free_block_queue", "self.null_block", "self.allocated_blocks", "self.null_block.block_id" ], "state_writes": [ "self.block_size", "self.max_blocks", "self.enable_caching", "self.blocks", "self.free_block_queue", "self.cached_block_hash_to_block", "self.hash_to_block", "self.request_tables", "self.allocated_blocks", "self.null_block", "self.null_block.is_null", "self.null_block.ref_count", "self.stats", "self._lock" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "allocate_block", "qualname": "PagedCacheManager.allocate_block", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.allocate_block", "kind": "method", "signature": "def allocate_block(self) -> Optional[CacheBlock]", "parameters": [], "return_annotation": "Optional[CacheBlock]", "docstring": "Allocate a new cache block.\n\nReturns:\n CacheBlock if available, None if out of memory.", "summary": "Allocate a new cache block.", "implementation": "Method `PagedCacheManager.allocate_block` updates `self.stats.allocated_blocks`, `self.stats.free_blocks`; calls `logger.warning`, `self.free_block_queue.popleft`, `self._maybe_evict_cached_block`, `block.touch`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 546, "end_line": 571, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L546-L571", "decorators": [], "calls": [ "logger.warning", "self.free_block_queue.popleft", "self._maybe_evict_cached_block", "block.touch" ], "state_reads": [ "self._lock", "self.free_block_queue.num_free_blocks", "self.free_block_queue", "self.free_block_queue.popleft", "self.enable_caching", "self._maybe_evict_cached_block", "self.allocated_blocks", "self.stats" ], "state_writes": [ "self.stats.allocated_blocks", "self.stats.free_blocks" ], "raises": [], "return_expressions": [ "None", "block" ], "awaits": false, "yields": false }, { "name": "get_new_blocks", "qualname": "PagedCacheManager.get_new_blocks", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_new_blocks", "kind": "method", "signature": "def get_new_blocks(self, num_blocks: int) -> List[CacheBlock]", "parameters": [ { "name": "num_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of blocks to allocate" } ], "return_annotation": "List[CacheBlock]", "docstring": "Allocate multiple blocks at once (vLLM style).\n\nArgs:\n num_blocks: Number of blocks to allocate\n\nReturns:\n List of allocated blocks\n\nRaises:\n ValueError: If not enough free blocks", "summary": "Allocate multiple blocks at once (vLLM style).", "implementation": "Method `PagedCacheManager.get_new_blocks` updates `self.stats.allocated_blocks`, `self.stats.free_blocks`; calls `ValueError`, `self.free_block_queue.popleft_n`, `self._maybe_evict_cached_block`, `block.touch`; can raise `ValueError`; returns `blocks`.", "documented": true, "public": true, "addressable": true, "line": 573, "end_line": 606, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L573-L606", "decorators": [], "calls": [ "ValueError", "self.free_block_queue.popleft_n", "self._maybe_evict_cached_block", "block.touch" ], "state_reads": [ "self._lock", "self.free_block_queue.num_free_blocks", "self.free_block_queue", "self.free_block_queue.popleft_n", "self.enable_caching", "self._maybe_evict_cached_block", "self.allocated_blocks", "self.stats" ], "state_writes": [ "self.stats.allocated_blocks", "self.stats.free_blocks" ], "raises": [ "ValueError" ], "return_expressions": [ "blocks" ], "awaits": false, "yields": false }, { "name": "_maybe_evict_cached_block", "qualname": "PagedCacheManager._maybe_evict_cached_block", "full_name": "vllm_mlx.paged_cache.PagedCacheManager._maybe_evict_cached_block", "kind": "method", "signature": "def _maybe_evict_cached_block(self, block: CacheBlock) -> bool", "parameters": [ { "name": "block", "kind": "positional or keyword", "annotation": "CacheBlock", "default": "", "required": true, "description": "Block to evict" } ], "return_annotation": "bool", "docstring": "Evict a block from the hash cache if present.\n\nArgs:\n block: Block to evict\n\nReturns:\n True if block was evicted from cache", "summary": "Evict a block from the hash cache if present.", "implementation": "Method `PagedCacheManager._maybe_evict_cached_block` updates `self.stats.evictions`; calls `self.cached_block_hash_to_block.pop`, `block.reset_hash`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 608, "end_line": 634, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L608-L634", "decorators": [], "calls": [ "self.cached_block_hash_to_block.pop", "block.reset_hash" ], "state_reads": [ "self.cached_block_hash_to_block.pop", "self.cached_block_hash_to_block", "self.hash_to_block", "self.stats" ], "state_writes": [ "self.stats.evictions" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "free_block", "qualname": "PagedCacheManager.free_block", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.free_block", "kind": "method", "signature": "def free_block(self, block_id: int) -> bool", "parameters": [ { "name": "block_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Free a cache block (decrements ref_count, frees if 0).\n\nReturns:\n True if block was freed, False if still referenced.", "summary": "Free a cache block (decrements ref_count, frees if 0).", "implementation": "Method `PagedCacheManager.free_block` updates `self.stats.allocated_blocks`, `self.stats.free_blocks`, `self.stats.total_tokens_cached`; calls `logger.warning`, `self.free_block_queue.append`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 636, "end_line": 667, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L636-L667", "decorators": [], "calls": [ "logger.warning", "self.free_block_queue.append" ], "state_reads": [ "self._lock", "self.allocated_blocks", "self.free_block_queue.append", "self.free_block_queue", "self.stats" ], "state_writes": [ "self.stats.allocated_blocks", "self.stats.free_blocks", "self.stats.total_tokens_cached" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "free_blocks", "qualname": "PagedCacheManager.free_blocks", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.free_blocks", "kind": "method", "signature": "def free_blocks(self, blocks: Iterable[CacheBlock]) -> None", "parameters": [ { "name": "blocks", "kind": "positional or keyword", "annotation": "Iterable[CacheBlock]", "default": "", "required": true, "description": "Blocks to free (in eviction order)" } ], "return_annotation": "None", "docstring": "Free multiple blocks (vLLM style).\n\nBlocks with ref_count=0 are added to the free queue.\n\nArgs:\n blocks: Blocks to free (in eviction order)", "summary": "Free multiple blocks (vLLM style).", "implementation": "Method `PagedCacheManager.free_blocks` updates `self.stats.allocated_blocks`, `self.stats.free_blocks`, `self.stats.total_tokens_cached`; calls `list`, `to_free.append`, `self.free_block_queue.append_n`.", "documented": true, "public": true, "addressable": true, "line": 669, "end_line": 696, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L669-L696", "decorators": [], "calls": [ "list", "to_free.append", "self.free_block_queue.append_n" ], "state_reads": [ "self._lock", "self.allocated_blocks", "self.stats", "self.free_block_queue.append_n", "self.free_block_queue" ], "state_writes": [ "self.stats.allocated_blocks", "self.stats.free_blocks", "self.stats.total_tokens_cached" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "touch", "qualname": "PagedCacheManager.touch", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.touch", "kind": "method", "signature": "def touch(self, blocks: Iterable[CacheBlock]) -> None", "parameters": [ { "name": "blocks", "kind": "positional or keyword", "annotation": "Iterable[CacheBlock]", "default": "", "required": true, "description": "Blocks to touch" } ], "return_annotation": "None", "docstring": "Touch blocks to prevent eviction (cache hit, vLLM style).\n\nIncrements ref_count and removes from free queue if needed.\n\nArgs:\n blocks: Blocks to touch", "summary": "Touch blocks to prevent eviction (cache hit, vLLM style).", "implementation": "Method `PagedCacheManager.touch` updates `self.stats.free_blocks`, `self.stats.allocated_blocks`; calls `self.free_block_queue.remove`, `block.touch`.", "documented": true, "public": true, "addressable": true, "line": 698, "end_line": 720, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L698-L720", "decorators": [], "calls": [ "self.free_block_queue.remove", "block.touch" ], "state_reads": [ "self._lock", "self.free_block_queue.remove", "self.free_block_queue", "self.stats", "self.allocated_blocks" ], "state_writes": [ "self.stats.free_blocks", "self.stats.allocated_blocks" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "increment_ref", "qualname": "PagedCacheManager.increment_ref", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.increment_ref", "kind": "method", "signature": "def increment_ref(self, block_id: int) -> bool", "parameters": [ { "name": "block_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Increment reference count for a block.", "summary": "Increment reference count for a block.", "implementation": "Method `PagedCacheManager.increment_ref` updates `self.stats.shared_blocks`; calls `block.touch`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 726, "end_line": 739, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L726-L739", "decorators": [], "calls": [ "block.touch" ], "state_reads": [ "self._lock", "self.allocated_blocks", "self.stats" ], "state_writes": [ "self.stats.shared_blocks" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "decrement_ref", "qualname": "PagedCacheManager.decrement_ref", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.decrement_ref", "kind": "method", "signature": "def decrement_ref(self, block_id: int) -> bool", "parameters": [ { "name": "block_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Decrement reference count (alias for free_block).", "summary": "Decrement reference count (alias for free_block).", "implementation": "Method `PagedCacheManager.decrement_ref` calls `self.free_block`; returns `self.free_block(block_id)`.", "documented": true, "public": true, "addressable": true, "line": 741, "end_line": 743, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L741-L743", "decorators": [], "calls": [ "self.free_block" ], "state_reads": [ "self.free_block" ], "state_writes": [], "raises": [], "return_expressions": [ "self.free_block(block_id)" ], "awaits": false, "yields": false }, { "name": "get_cached_block", "qualname": "PagedCacheManager.get_cached_block", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_cached_block", "kind": "method", "signature": "def get_cached_block(self, block_hash: BlockHash) -> Optional[CacheBlock]", "parameters": [ { "name": "block_hash", "kind": "positional or keyword", "annotation": "BlockHash", "default": "", "required": true, "description": "Content hash of the block" } ], "return_annotation": "Optional[CacheBlock]", "docstring": "Get a cached block by its hash (vLLM style).\n\nArgs:\n block_hash: Content hash of the block\n\nReturns:\n Cached block if found, None otherwise", "summary": "Get a cached block by its hash (vLLM style).", "implementation": "Method `PagedCacheManager.get_cached_block` updates `self.stats.cache_hits`, `self.stats.cache_misses`; calls `self.cached_block_hash_to_block.get_block`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 749, "end_line": 768, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L749-L768", "decorators": [], "calls": [ "self.cached_block_hash_to_block.get_block" ], "state_reads": [ "self.enable_caching", "self._lock", "self.cached_block_hash_to_block.get_block", "self.cached_block_hash_to_block", "self.stats" ], "state_writes": [ "self.stats.cache_hits", "self.stats.cache_misses" ], "raises": [], "return_expressions": [ "None", "block" ], "awaits": false, "yields": false }, { "name": "cache_full_blocks", "qualname": "PagedCacheManager.cache_full_blocks", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.cache_full_blocks", "kind": "method", "signature": "def cache_full_blocks(self, blocks: List[CacheBlock], token_ids: List[int], num_cached_blocks: int, num_full_blocks: int) -> None", "parameters": [ { "name": "blocks", "kind": "positional or keyword", "annotation": "List[CacheBlock]", "default": "", "required": true, "description": "All blocks for the request" }, { "name": "token_ids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "All token IDs for the request" }, { "name": "num_cached_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of blocks already cached" }, { "name": "num_full_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Number of full blocks to cache" } ], "return_annotation": "None", "docstring": "Cache full blocks for prefix caching (vLLM style).\n\nComputes chain hashes and adds blocks to the cache.\n\nArgs:\n blocks: All blocks for the request\n token_ids: All token IDs for the request\n num_cached_blocks: Number of blocks already cached\n num_full_blocks: Number of full blocks to cache", "summary": "Cache full blocks for prefix caching (vLLM style).", "implementation": "Method `PagedCacheManager.cache_full_blocks` calls `range`, `compute_block_hash`, `len`, `self.cached_block_hash_to_block.insert`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 770, "end_line": 824, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L770-L824", "decorators": [], "calls": [ "range", "compute_block_hash", "len", "self.cached_block_hash_to_block.insert", "self.compute_block_hash" ], "state_reads": [ "self.enable_caching", "self._lock", "self.block_size", "self.cached_block_hash_to_block.insert", "self.cached_block_hash_to_block", "self.compute_block_hash", "self.hash_to_block" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_computed_blocks", "qualname": "PagedCacheManager.get_computed_blocks", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_computed_blocks", "kind": "method", "signature": "def get_computed_blocks(self, token_ids: List[int]) -> Tuple[List[CacheBlock], int]", "parameters": [ { "name": "token_ids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Token IDs to look up" } ], "return_annotation": "Tuple[List[CacheBlock], int]", "docstring": "Find cached blocks for a token prefix (vLLM style).\n\nArgs:\n token_ids: Token IDs to look up\n\nReturns:\n Tuple of (cached_blocks, num_cached_tokens)", "summary": "Find cached blocks for a token prefix (vLLM style).", "implementation": "Method `PagedCacheManager.get_computed_blocks` updates `self.stats.cache_misses`, `self.stats.cache_hits`; calls `len`, `range`, `compute_block_hash`, `self.cached_block_hash_to_block.get_block`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 826, "end_line": 868, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L826-L868", "decorators": [], "calls": [ "len", "range", "compute_block_hash", "self.cached_block_hash_to_block.get_block", "cached_blocks.append" ], "state_reads": [ "self.enable_caching", "self._lock", "self.block_size", "self.cached_block_hash_to_block.get_block", "self.cached_block_hash_to_block", "self.stats" ], "state_writes": [ "self.stats.cache_misses", "self.stats.cache_hits" ], "raises": [], "return_expressions": [ "([], 0)", "(cached_blocks, num_cached_tokens)" ], "awaits": false, "yields": false }, { "name": "compute_block_hash", "qualname": "PagedCacheManager.compute_block_hash", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.compute_block_hash", "kind": "method", "signature": "def compute_block_hash(tokens: List[int]) -> str", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Compute legacy string hash for a sequence of tokens.", "summary": "Compute legacy string hash for a sequence of tokens.", "implementation": "Method `PagedCacheManager.compute_block_hash` calls `b''.join`, `t.to_bytes`, `hashlib.sha256(token_bytes).hexdigest`, `hashlib.sha256`; returns `hashlib.sha256(token_bytes).hexdigest()[:16]`.", "documented": true, "public": true, "addressable": true, "line": 875, "end_line": 878, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L875-L878", "decorators": [ "staticmethod" ], "calls": [ "b''.join", "t.to_bytes", "hashlib.sha256(token_bytes).hexdigest", "hashlib.sha256" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hashlib.sha256(token_bytes).hexdigest()[:16]" ], "awaits": false, "yields": false }, { "name": "find_cached_block", "qualname": "PagedCacheManager.find_cached_block", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.find_cached_block", "kind": "method", "signature": "def find_cached_block(self, tokens: List[int]) -> Optional[CacheBlock]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[CacheBlock]", "docstring": "Find a cached block matching the given tokens (legacy method).", "summary": "Find a cached block matching the given tokens (legacy method).", "implementation": "Method `PagedCacheManager.find_cached_block` updates `self.stats.cache_hits`, `self.stats.cache_misses`; calls `self.compute_block_hash`, `block.touch`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 880, "end_line": 896, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L880-L896", "decorators": [], "calls": [ "self.compute_block_hash", "block.touch" ], "state_reads": [ "self._lock", "self.compute_block_hash", "self.hash_to_block", "self.allocated_blocks", "self.stats" ], "state_writes": [ "self.stats.cache_hits", "self.stats.cache_misses" ], "raises": [], "return_expressions": [ "block", "None" ], "awaits": false, "yields": false }, { "name": "register_block_hash", "qualname": "PagedCacheManager.register_block_hash", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.register_block_hash", "kind": "method", "signature": "def register_block_hash(self, block: CacheBlock, tokens: List[int]) -> None", "parameters": [ { "name": "block", "kind": "positional or keyword", "annotation": "CacheBlock", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Register a block's hash for deduplication (legacy method).", "summary": "Register a block's hash for deduplication (legacy method).", "implementation": "Method `PagedCacheManager.register_block_hash` calls `self.compute_block_hash`.", "documented": true, "public": true, "addressable": true, "line": 898, "end_line": 903, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L898-L903", "decorators": [], "calls": [ "self.compute_block_hash" ], "state_reads": [ "self._lock", "self.compute_block_hash", "self.hash_to_block" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "create_block_table", "qualname": "PagedCacheManager.create_block_table", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.create_block_table", "kind": "method", "signature": "def create_block_table(self, request_id: str) -> BlockTable", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BlockTable", "docstring": "Create a new block table for a request.", "summary": "Create a new block table for a request.", "implementation": "Method `PagedCacheManager.create_block_table` calls `BlockTable`; returns `table`.", "documented": true, "public": true, "addressable": true, "line": 909, "end_line": 914, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L909-L914", "decorators": [], "calls": [ "BlockTable" ], "state_reads": [ "self._lock", "self.request_tables" ], "state_writes": [], "raises": [], "return_expressions": [ "table" ], "awaits": false, "yields": false }, { "name": "get_block_table", "qualname": "PagedCacheManager.get_block_table", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_block_table", "kind": "method", "signature": "def get_block_table(self, request_id: str) -> Optional[BlockTable]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[BlockTable]", "docstring": "Get block table for a request.", "summary": "Get block table for a request.", "implementation": "Method `PagedCacheManager.get_block_table` calls `self.request_tables.get`; returns `self.request_tables.get(request_id)`.", "documented": true, "public": true, "addressable": true, "line": 916, "end_line": 919, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L916-L919", "decorators": [], "calls": [ "self.request_tables.get" ], "state_reads": [ "self._lock", "self.request_tables.get", "self.request_tables" ], "state_writes": [], "raises": [], "return_expressions": [ "self.request_tables.get(request_id)" ], "awaits": false, "yields": false }, { "name": "get_or_create_block_table", "qualname": "PagedCacheManager.get_or_create_block_table", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_or_create_block_table", "kind": "method", "signature": "def get_or_create_block_table(self, request_id: str) -> BlockTable", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BlockTable", "docstring": "Get or create block table for a request.", "summary": "Get or create block table for a request.", "implementation": "Method `PagedCacheManager.get_or_create_block_table` calls `BlockTable`; returns `self.request_tables[request_id]`.", "documented": true, "public": true, "addressable": true, "line": 921, "end_line": 926, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L921-L926", "decorators": [], "calls": [ "BlockTable" ], "state_reads": [ "self._lock", "self.request_tables" ], "state_writes": [], "raises": [], "return_expressions": [ "self.request_tables[request_id]" ], "awaits": false, "yields": false }, { "name": "delete_block_table", "qualname": "PagedCacheManager.delete_block_table", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.delete_block_table", "kind": "method", "signature": "def delete_block_table(self, request_id: str) -> None", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Delete block table and free associated blocks.", "summary": "Delete block table and free associated blocks.", "implementation": "Method `PagedCacheManager.delete_block_table` calls `self.request_tables.pop`, `self.free_block`.", "documented": true, "public": true, "addressable": true, "line": 928, "end_line": 934, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L928-L934", "decorators": [], "calls": [ "self.request_tables.pop", "self.free_block" ], "state_reads": [ "self._lock", "self.request_tables.pop", "self.request_tables", "self.free_block" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "add_block_to_table", "qualname": "PagedCacheManager.add_block_to_table", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.add_block_to_table", "kind": "method", "signature": "def add_block_to_table(self, table: BlockTable, block: CacheBlock, tokens_in_block: int) -> None", "parameters": [ { "name": "table", "kind": "positional or keyword", "annotation": "BlockTable", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "block", "kind": "positional or keyword", "annotation": "CacheBlock", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens_in_block", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Add a block to a block table.", "summary": "Add a block to a block table.", "implementation": "Method `PagedCacheManager.add_block_to_table` updates `self.stats.total_tokens_cached`; calls `table.block_ids.append`.", "documented": true, "public": true, "addressable": true, "line": 936, "end_line": 947, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L936-L947", "decorators": [], "calls": [ "table.block_ids.append" ], "state_reads": [ "self._lock", "self.stats" ], "state_writes": [ "self.stats.total_tokens_cached" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "find_shared_prefix", "qualname": "PagedCacheManager.find_shared_prefix", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.find_shared_prefix", "kind": "method", "signature": "def find_shared_prefix(self, tokens: List[int]) -> Tuple[List[int], List[int]]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Tuple[List[int], List[int]]", "docstring": "Find shared prefix blocks for a token sequence.", "summary": "Find shared prefix blocks for a token sequence.", "implementation": "Method `PagedCacheManager.find_shared_prefix` calls `tokens.copy`, `len`, `self.find_cached_block`, `shared_blocks.append`; returns `(shared_blocks, remaining_tokens)`.", "documented": true, "public": true, "addressable": true, "line": 953, "end_line": 974, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L953-L974", "decorators": [], "calls": [ "tokens.copy", "len", "self.find_cached_block", "shared_blocks.append" ], "state_reads": [ "self._lock", "self.block_size", "self.find_cached_block" ], "state_writes": [], "raises": [], "return_expressions": [ "(shared_blocks, remaining_tokens)" ], "awaits": false, "yields": false }, { "name": "fork_block_table", "qualname": "PagedCacheManager.fork_block_table", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.fork_block_table", "kind": "method", "signature": "def fork_block_table(self, source_table: BlockTable, new_request_id: str) -> BlockTable", "parameters": [ { "name": "source_table", "kind": "positional or keyword", "annotation": "BlockTable", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "new_request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BlockTable", "docstring": "Fork a block table for a new request (COW).", "summary": "Fork a block table for a new request (COW).", "implementation": "Method `PagedCacheManager.fork_block_table` calls `source_table.copy`, `self.increment_ref`, `logger.debug`, `len`; returns `new_table`.", "documented": true, "public": true, "addressable": true, "line": 976, "end_line": 997, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L976-L997", "decorators": [], "calls": [ "source_table.copy", "self.increment_ref", "logger.debug", "len" ], "state_reads": [ "self._lock", "self.increment_ref", "self.request_tables" ], "state_writes": [], "raises": [], "return_expressions": [ "new_table" ], "awaits": false, "yields": false }, { "name": "get_blocks_for_generation", "qualname": "PagedCacheManager.get_blocks_for_generation", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_blocks_for_generation", "kind": "method", "signature": "def get_blocks_for_generation(self, table: BlockTable) -> Tuple[List[CacheBlock], bool]", "parameters": [ { "name": "table", "kind": "positional or keyword", "annotation": "BlockTable", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Tuple[List[CacheBlock], bool]", "docstring": "Get blocks for generation, applying COW if needed.", "summary": "Get blocks for generation, applying COW if needed.", "implementation": "Method `PagedCacheManager.get_blocks_for_generation` updates `self.stats.cow_copies`; calls `enumerate`, `self.allocated_blocks.get`, `block.is_shared`, `self._cow_copy_block`; returns `(blocks, was_copied)`.", "documented": true, "public": true, "addressable": true, "line": 999, "end_line": 1029, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L999-L1029", "decorators": [], "calls": [ "enumerate", "self.allocated_blocks.get", "block.is_shared", "self._cow_copy_block", "blocks.append", "block.touch" ], "state_reads": [ "self._lock", "self.allocated_blocks.get", "self.allocated_blocks", "self._cow_copy_block", "self.stats" ], "state_writes": [ "self.stats.cow_copies" ], "raises": [], "return_expressions": [ "(blocks, was_copied)" ], "awaits": false, "yields": false }, { "name": "_cow_copy_block", "qualname": "PagedCacheManager._cow_copy_block", "full_name": "vllm_mlx.paged_cache.PagedCacheManager._cow_copy_block", "kind": "method", "signature": "def _cow_copy_block(self, source_block: CacheBlock) -> Optional[CacheBlock]", "parameters": [ { "name": "source_block", "kind": "positional or keyword", "annotation": "CacheBlock", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[CacheBlock]", "docstring": "Create a copy of a block for COW.", "summary": "Create a copy of a block for COW.", "implementation": "Method `PagedCacheManager._cow_copy_block` updates `self.stats.shared_blocks`; calls `self.allocate_block`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1031, "end_line": 1046, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1031-L1046", "decorators": [], "calls": [ "self.allocate_block", "logger.debug" ], "state_reads": [ "self.allocate_block", "self.stats" ], "state_writes": [ "self.stats.shared_blocks" ], "raises": [], "return_expressions": [ "None", "new_block" ], "awaits": false, "yields": false }, { "name": "allocate_blocks_for_tokens", "qualname": "PagedCacheManager.allocate_blocks_for_tokens", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.allocate_blocks_for_tokens", "kind": "method", "signature": "def allocate_blocks_for_tokens(self, num_tokens: int) -> List[CacheBlock]", "parameters": [ { "name": "num_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "List[CacheBlock]", "docstring": "Allocate enough blocks to hold num_tokens.", "summary": "Allocate enough blocks to hold num_tokens.", "implementation": "Method `PagedCacheManager.allocate_blocks_for_tokens` calls `self.get_new_blocks`; returns `self.get_new_blocks(num_blocks_needed)`.", "documented": true, "public": true, "addressable": true, "line": 1052, "end_line": 1055, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1052-L1055", "decorators": [], "calls": [ "self.get_new_blocks" ], "state_reads": [ "self.block_size", "self.get_new_blocks" ], "state_writes": [], "raises": [], "return_expressions": [ "self.get_new_blocks(num_blocks_needed)" ], "awaits": false, "yields": false }, { "name": "evict_lru_blocks", "qualname": "PagedCacheManager.evict_lru_blocks", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.evict_lru_blocks", "kind": "method", "signature": "def evict_lru_blocks(self, num_blocks: int) -> int", "parameters": [ { "name": "num_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Evict least recently used blocks.\n\nWith the doubly linked list, LRU blocks are already at the front\nof the free queue. We just need to pop from front.", "summary": "Evict least recently used blocks.", "implementation": "Method `PagedCacheManager.evict_lru_blocks` calls `range`, `min`, `self.free_block_queue.popleft`, `self._maybe_evict_cached_block`; returns `evicted`.", "documented": true, "public": true, "addressable": true, "line": 1061, "end_line": 1085, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1061-L1085", "decorators": [], "calls": [ "range", "min", "self.free_block_queue.popleft", "self._maybe_evict_cached_block", "self.free_block_queue.append", "logger.info" ], "state_reads": [ "self._lock", "self.free_block_queue.num_free_blocks", "self.free_block_queue", "self.free_block_queue.popleft", "self._maybe_evict_cached_block", "self.free_block_queue.append" ], "state_writes": [], "raises": [], "return_expressions": [ "evicted" ], "awaits": false, "yields": false }, { "name": "handle_memory_pressure", "qualname": "PagedCacheManager.handle_memory_pressure", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.handle_memory_pressure", "kind": "method", "signature": "def handle_memory_pressure(self, requested_blocks: int) -> bool", "parameters": [ { "name": "requested_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Handle memory pressure by evicting blocks.", "summary": "Handle memory pressure by evicting blocks.", "implementation": "Method `PagedCacheManager.handle_memory_pressure` calls `self.evict_lru_blocks`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1087, "end_line": 1096, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1087-L1096", "decorators": [], "calls": [ "self.evict_lru_blocks" ], "state_reads": [ "self._lock", "self.free_block_queue.num_free_blocks", "self.free_block_queue", "self.evict_lru_blocks" ], "state_writes": [], "raises": [], "return_expressions": [ "True", "self.free_block_queue.num_free_blocks >= requested_blocks" ], "awaits": false, "yields": false }, { "name": "free_blocks", "qualname": "PagedCacheManager.free_blocks", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.free_blocks", "kind": "method", "signature": "def free_blocks(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Number of free blocks available.", "summary": "Number of free blocks available.", "implementation": "Method `PagedCacheManager.free_blocks` returns `self.free_block_queue.num_free_blocks`.", "documented": true, "public": true, "addressable": true, "line": 1103, "end_line": 1105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1103-L1105", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.free_block_queue.num_free_blocks", "self.free_block_queue" ], "state_writes": [], "raises": [], "return_expressions": [ "self.free_block_queue.num_free_blocks" ], "awaits": false, "yields": false }, { "name": "usage", "qualname": "PagedCacheManager.usage", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.usage", "kind": "method", "signature": "def usage(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Cache usage ratio (0.0 to 1.0).", "summary": "Cache usage ratio (0.0 to 1.0).", "implementation": "Method `PagedCacheManager.usage` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1108, "end_line": 1113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1108-L1113", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.max_blocks", "self.free_blocks" ], "state_writes": [], "raises": [], "return_expressions": [ "0.0", "1.0 - self.free_blocks / total" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "PagedCacheManager.get_stats", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_stats", "kind": "method", "signature": "def get_stats(self) -> CacheStats", "parameters": [], "return_annotation": "CacheStats", "docstring": "Get current cache statistics.", "summary": "Get current cache statistics.", "implementation": "Method `PagedCacheManager.get_stats` updates `self.stats.shared_blocks`, `self.stats.free_blocks`; calls `sum`, `self.allocated_blocks.values`; returns `self.stats`.", "documented": true, "public": true, "addressable": true, "line": 1115, "end_line": 1122, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1115-L1122", "decorators": [], "calls": [ "sum", "self.allocated_blocks.values" ], "state_reads": [ "self._lock", "self.stats", "self.allocated_blocks.values", "self.allocated_blocks", "self.free_block_queue.num_free_blocks", "self.free_block_queue" ], "state_writes": [ "self.stats.shared_blocks", "self.stats.free_blocks" ], "raises": [], "return_expressions": [ "self.stats" ], "awaits": false, "yields": false }, { "name": "get_memory_usage", "qualname": "PagedCacheManager.get_memory_usage", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.get_memory_usage", "kind": "method", "signature": "def get_memory_usage(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get memory usage information.", "summary": "Get memory usage information.", "implementation": "Method `PagedCacheManager.get_memory_usage` calls `self.get_stats`; returns `{'block_size': self.block_size, 'max_blocks': self.max_blocks, 'allocated_blocks': stats.allocated_blocks, 'free_blocks…`.", "documented": true, "public": true, "addressable": true, "line": 1124, "end_line": 1141, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1124-L1141", "decorators": [], "calls": [ "self.get_stats" ], "state_reads": [ "self._lock", "self.get_stats", "self.block_size", "self.max_blocks" ], "state_writes": [], "raises": [], "return_expressions": [ "{'block_size': self.block_size, 'max_blocks': self.max_blocks, 'allocated_blocks': stats.allocated_blocks, 'free_blocks…" ], "awaits": false, "yields": false }, { "name": "reset_stats", "qualname": "PagedCacheManager.reset_stats", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.reset_stats", "kind": "method", "signature": "def reset_stats(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset statistics counters.", "summary": "Reset statistics counters.", "implementation": "Method `PagedCacheManager.reset_stats` updates `self.stats.cache_hits`, `self.stats.cache_misses`, `self.stats.cow_copies`, `self.stats.evictions`.", "documented": true, "public": true, "addressable": true, "line": 1143, "end_line": 1149, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1143-L1149", "decorators": [], "calls": [], "state_reads": [ "self._lock", "self.stats" ], "state_writes": [ "self.stats.cache_hits", "self.stats.cache_misses", "self.stats.cow_copies", "self.stats.evictions" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset_prefix_cache", "qualname": "PagedCacheManager.reset_prefix_cache", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.reset_prefix_cache", "kind": "method", "signature": "def reset_prefix_cache(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Reset the prefix cache.", "summary": "Reset the prefix cache.", "implementation": "Method `PagedCacheManager.reset_prefix_cache` updates `self.stats.evictions`, `self.stats.cache_hits`, `self.stats.cache_misses`; calls `logger.warning`, `self.cached_block_hash_to_block.clear`, `self.hash_to_block.clear`, `block.reset_hash`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1151, "end_line": 1171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1151-L1171", "decorators": [], "calls": [ "logger.warning", "self.cached_block_hash_to_block.clear", "self.hash_to_block.clear", "block.reset_hash", "logger.info" ], "state_reads": [ "self._lock", "self.max_blocks", "self.free_block_queue.num_free_blocks", "self.free_block_queue", "self.cached_block_hash_to_block.clear", "self.cached_block_hash_to_block", "self.hash_to_block.clear", "self.hash_to_block", "self.blocks", "self.stats" ], "state_writes": [ "self.stats.evictions", "self.stats.cache_hits", "self.stats.cache_misses" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "clear", "qualname": "PagedCacheManager.clear", "full_name": "vllm_mlx.paged_cache.PagedCacheManager.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear all cached data.", "summary": "Clear all cached data.", "implementation": "Method `PagedCacheManager.clear` updates `self.blocks`, `self.free_block_queue`, `self.null_block`, `self.null_block.is_null`; calls `CacheBlock`, `range`, `FreeKVCacheBlockQueue`, `self.cached_block_hash_to_block.clear`.", "documented": true, "public": true, "addressable": true, "line": 1173, "end_line": 1197, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/paged_cache.py#L1173-L1197", "decorators": [], "calls": [ "CacheBlock", "range", "FreeKVCacheBlockQueue", "self.cached_block_hash_to_block.clear", "self.hash_to_block.clear", "self.request_tables.clear", "self.allocated_blocks.clear", "self.free_block_queue.popleft", "CacheStats", "logger.info" ], "state_reads": [ "self._lock", "self.max_blocks", "self.blocks", "self.cached_block_hash_to_block.clear", "self.cached_block_hash_to_block", "self.hash_to_block.clear", "self.hash_to_block", "self.request_tables.clear", "self.request_tables", "self.allocated_blocks.clear", "self.allocated_blocks", "self.free_block_queue.popleft", "self.free_block_queue", "self.null_block", "self.null_block.block_id" ], "state_writes": [ "self.blocks", "self.free_block_queue", "self.null_block", "self.null_block.is_null", "self.null_block.ref_count", "self.stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.patches", "path": "vllm_mlx/patches/__init__.py", "page_path": "reference/api/vllm_mlx/patches/index.md", "docstring": "Narrow runtime compatibility patches for supported model architectures.", "summary": "Narrow runtime compatibility patches for supported model architectures.", "line_count": 2, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/__init__.py#L1-L2", "members": [], "symbols": [] }, { "name": "vllm_mlx.patches.gemma4_mllm", "path": "vllm_mlx/patches/gemma4_mllm.py", "page_path": "reference/api/vllm_mlx/patches/gemma4_mllm.md", "docstring": "Runtime patch for mlx-vlm Gemma 4 Attention to trim oversized masks.\n\nmlx-vlm 0.5.0's stock Gemma 4 attention assumes the mask's last dim matches\nkeys.shape[-2] exactly. vllm-mlx's BatchedEngine MLLM path (continuous\nbatching) sometimes passes a mask sized for the max sequence in the batch\nwhile a specific layer's keys end up shorter — sliding-window layers cap\nkeys at window=512, the mask is built once for the full prompt. Without a\ntrim, scaled_dot_product_attention sees a shape mismatch.\n\nThat mask trim is the only behavior this patch adds; everything else\nmirrors mlx-vlm 0.5.0 verbatim (signature, return shape, offset handling).\nThe previous reason for this patch — BatchKVCache's in-place `+=` on\n`cache.offset` corrupting RoPE — is now handled upstream: mlx-vlm 0.5.0\nline 223 does `offset = mx.array(cache.offset) if cache is not None else 0`\nwhich is a defensive copy. (Confirmed in review of PR #564.)", "summary": "Runtime patch for mlx-vlm Gemma 4 Attention to trim oversized masks.", "line_count": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/gemma4_mllm.py#L1-L98", "members": [ "logger", "patch_gemma4_attention_for_batching" ], "symbols": [ { "name": "patch_gemma4_attention_for_batching", "qualname": "patch_gemma4_attention_for_batching", "full_name": "vllm_mlx.patches.gemma4_mllm.patch_gemma4_attention_for_batching", "kind": "function", "signature": "def patch_gemma4_attention_for_batching() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Patch Gemma 4 Attention.__call__ to trim oversized masks.\n\nOtherwise mirrors mlx-vlm 0.5.0 upstream verbatim. Returns True if\napplied, False if mlx-vlm is not installed or Gemma 4 unavailable.", "summary": "Patch Gemma 4 Attention.__call__ to trim oversized masks.", "implementation": "Function `patch_gemma4_attention_for_batching` calls `logger.debug`, `getattr`, `logger.info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 98, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/gemma4_mllm.py#L28-L98", "decorators": [], "calls": [ "logger.debug", "getattr", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_patched_call", "qualname": "patch_gemma4_attention_for_batching._patched_call", "full_name": "vllm_mlx.patches.gemma4_mllm.patch_gemma4_attention_for_batching._patched_call", "kind": "nested function", "signature": "def _patched_call(self, x: mx.array, mask: Optional[mx.array]=None, cache: Optional[Any]=None, shared_kv: Optional[tuple]=None, offset: Optional[Any]=None) -> Any", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "mask", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "cache", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "shared_kv", "kind": "positional or keyword", "annotation": "Optional[tuple]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "offset", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "Any", "docstring": "", "summary": "Nested Function `patch_gemma4_attention_for_batching._patched_call` calls `self.q_proj(x).reshape`, `self.q_proj`, `self.q_norm`, `self.k_proj(x).reshape`; returns `(self.o_proj(output), (keys, values), offset)`.", "implementation": "Nested Function `patch_gemma4_attention_for_batching._patched_call` calls `self.q_proj(x).reshape`, `self.q_proj`, `self.q_norm`, `self.k_proj(x).reshape`; returns `(self.o_proj(output), (keys, values), offset)`.", "documented": false, "public": false, "addressable": false, "line": 45, "end_line": 93, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/gemma4_mllm.py#L45-L93", "decorators": [], "calls": [ "self.q_proj(x).reshape", "self.q_proj", "self.q_norm", "self.k_proj(x).reshape", "self.k_proj", "self.v_proj(x).reshape", "self.v_proj", "mx.array", "self.k_norm", "keys.transpose", "self.rope", "self.v_norm", "values.transpose", "cache.update_and_fetch", "queries.transpose", "isinstance", "scaled_dot_product_attention", "output.transpose(0, 2, 1, 3).reshape", "output.transpose", "self.o_proj" ], "state_reads": [ "self.q_proj", "self.n_heads", "self.head_dim", "self.q_norm", "self.k_proj", "self.n_kv_heads", "self.use_k_eq_v", "self.v_proj", "self.k_norm", "self.rope", "self.v_norm", "self.scale", "self.o_proj" ], "state_writes": [], "raises": [], "return_expressions": [ "(self.o_proj(output), (keys, values), offset)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.patches.glm4v_moe_mllm", "path": "vllm_mlx/patches/glm4v_moe_mllm.py", "page_path": "reference/api/vllm_mlx/patches/glm4v_moe_mllm.md", "docstring": "Runtime patch for mlx-vlm's GLM-4.6V model to support BatchKVCache.\n\nGLM-4.6V (glm4v_moe) computes position_ids from cache[0].offset once at\nthe start of GLM4VModel.__call__, then derives position_embeddings used\nby ALL decoder layers:\n\n position_ids = mx.arange(cache[0].offset, cache[0].offset + seq_len)\n position_embeddings = self.rotary_emb(h, position_ids)\n for layer in self.layers:\n h = layer(h, mask, cache[i], position_embeddings)\n\nFor regular KVCache, cache.offset is a Python int, so mx.arange works fine.\nFor BatchKVCache, cache.offset is an mx.array (per-batch-item offsets), and\nmx.arange does not support mx.array start/stop arguments, producing wrong\nposition_ids that corrupt RoPE embeddings for ALL layers.\n\nThis patch replaces GLM4VModel.__call__ with a version that converts\ncache[0].offset to int before computing position_ids.", "summary": "Runtime patch for mlx-vlm's GLM-4.6V model to support BatchKVCache.", "line_count": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/glm4v_moe_mllm.py#L1-L89", "members": [ "logger", "patch_glm4v_moe_for_batching" ], "symbols": [ { "name": "patch_glm4v_moe_for_batching", "qualname": "patch_glm4v_moe_for_batching", "full_name": "vllm_mlx.patches.glm4v_moe_mllm.patch_glm4v_moe_for_batching", "kind": "function", "signature": "def patch_glm4v_moe_for_batching() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Monkey-patch GLM4VModel.__call__ to handle BatchKVCache offset.\n\nReturns True if patch was applied, False if mlx-vlm is not installed\nor GLM-4.6V module not available.", "summary": "Monkey-patch GLM4VModel.__call__ to handle BatchKVCache offset.", "implementation": "Function `patch_glm4v_moe_for_batching` calls `logger.debug`, `getattr`, `logger.info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 31, "end_line": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/glm4v_moe_mllm.py#L31-L89", "decorators": [], "calls": [ "logger.debug", "getattr", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_patched_call", "qualname": "patch_glm4v_moe_for_batching._patched_call", "full_name": "vllm_mlx.patches.glm4v_moe_mllm.patch_glm4v_moe_for_batching._patched_call", "kind": "nested function", "signature": "def _patched_call(self, inputs: mx.array, inputs_embeds: Optional[mx.array]=None, cache: Optional[Any]=None, mask: Optional[mx.array]=None, position_ids: Optional[mx.array]=None) -> mx.array", "parameters": [ { "name": "inputs", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "inputs_embeds", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "cache", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "mask", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "position_ids", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "mx.array", "docstring": "", "summary": "Nested Function `patch_glm4v_moe_for_batching._patched_call` calls `self.embed_tokens`, `inputs_embeds.astype`, `isinstance`, `int`; returns `self.norm(h)`.", "implementation": "Nested Function `patch_glm4v_moe_for_batching._patched_call` calls `self.embed_tokens`, `inputs_embeds.astype`, `isinstance`, `int`; returns `self.norm(h)`.", "documented": false, "public": false, "addressable": false, "line": 50, "end_line": 84, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/glm4v_moe_mllm.py#L50-L84", "decorators": [], "calls": [ "self.embed_tokens", "inputs_embeds.astype", "isinstance", "int", "offset.max().item", "offset.max", "mx.arange", "mx.expand_dims", "mx.tile", "self.rotary_emb", "create_attention_mask", "range", "self.layers[self.start_idx + i]", "self.norm" ], "state_reads": [ "self.embed_tokens", "self.norm.weight.dtype", "self.norm.weight", "self.norm", "self.rotary_emb", "self.num_layers", "self.layers", "self.start_idx" ], "state_writes": [], "raises": [], "return_expressions": [ "self.norm(h)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.patches.qwen3_5_mllm", "path": "vllm_mlx/patches/qwen3_5_mllm.py", "page_path": "reference/api/vllm_mlx/patches/qwen3_5_mllm.md", "docstring": "Runtime patch for mlx-vlm's Qwen3.5 attention to support BatchKVCache.\n\nQwen 3.6 artifacts use the mlx-vlm Qwen3.5 language module in this stack.\nThe attention patch therefore lives in the Qwen3.5 compatibility module while\nserving Qwen 3.6 27B/35B/122B artifacts.\n\nmlx-vlm's Qwen3_5Attention uses cache.offset directly for kv_seq_len\ncomputation and mask slicing. BatchKVCache stores offset as mx.array\n(per-batch-item), not int, causing:\n\n mask = mask[..., :kv_seq_len]\n ValueError: Slice indices must be integers or None.\n\nThis patch replaces Qwen3_5Attention.__call__ with a version that converts\ncache.offset to int before using it for arithmetic/slicing, while leaving the\nactual cache.offset untouched so update_and_fetch still works correctly with\nper-batch offsets.", "summary": "Runtime patch for mlx-vlm's Qwen3.5 attention to support BatchKVCache.", "line_count": 266, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L1-L266", "members": [ "logger", "_cache_offset_to_int", "_default_target_verify_linears", "_default_target_verify_left_padded_attention", "_normalize_position_inputs", "_position_ids_for_offset", "_kv_seq_len", "_apply_rotary", "_slice_attention_mask", "_maybe_target_verify_attention", "patch_qwen35_attention_for_batching" ], "symbols": [ { "name": "_cache_offset_to_int", "qualname": "_cache_offset_to_int", "full_name": "vllm_mlx.patches.qwen3_5_mllm._cache_offset_to_int", "kind": "function", "signature": "def _cache_offset_to_int(cache) -> int", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Extract cache offset as int, handling BatchKVCache mx.array offset.", "summary": "Extract cache offset as int, handling BatchKVCache mx.array offset.", "implementation": "Function `_cache_offset_to_int` calls `isinstance`, `int`, `off.max().item`, `off.max`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 33, "end_line": 42, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L33-L42", "decorators": [], "calls": [ "isinstance", "int", "off.max().item", "off.max", "off.item" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0", "off", "int(off.max().item()) if off.ndim > 0 else int(off.item())", "int(off)" ], "awaits": false, "yields": false }, { "name": "_default_target_verify_linears", "qualname": "_default_target_verify_linears", "full_name": "vllm_mlx.patches.qwen3_5_mllm._default_target_verify_linears", "kind": "function", "signature": "def _default_target_verify_linears(linears, x, target_verify: bool)", "parameters": [ { "name": "linears", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "target_verify", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Function `_default_target_verify_linears` calls `tuple`, `linear`; returns `tuple((linear(x) for linear in linears))`.", "implementation": "Function `_default_target_verify_linears` calls `tuple`, `linear`; returns `tuple((linear(x) for linear in linears))`.", "documented": false, "public": false, "addressable": true, "line": 45, "end_line": 46, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L45-L46", "decorators": [], "calls": [ "tuple", "linear" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tuple((linear(x) for linear in linears))" ], "awaits": false, "yields": false }, { "name": "_default_target_verify_left_padded_attention", "qualname": "_default_target_verify_left_padded_attention", "full_name": "vllm_mlx.patches.qwen3_5_mllm._default_target_verify_left_padded_attention", "kind": "function", "signature": "def _default_target_verify_left_padded_attention(*args, **kwargs)", "parameters": [ { "name": "*args", "kind": "variadic positional", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic positional inputs accepted by this callable." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Function `_default_target_verify_left_padded_attention` returns `None`.", "implementation": "Function `_default_target_verify_left_padded_attention` returns `None`.", "documented": false, "public": false, "addressable": true, "line": 49, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L49-L50", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_normalize_position_inputs", "qualname": "_normalize_position_inputs", "full_name": "vllm_mlx.patches.qwen3_5_mllm._normalize_position_inputs", "kind": "function", "signature": "def _normalize_position_inputs(position_ids: Optional[mx.array], position_embeddings: Optional[tuple[mx.array, mx.array]], length: int) -> tuple[Optional[mx.array], Optional[tuple[mx.array, mx.array]]]", "parameters": [ { "name": "position_ids", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "position_embeddings", "kind": "positional or keyword", "annotation": "Optional[tuple[mx.array, mx.array]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "length", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[Optional[mx.array], Optional[tuple[mx.array, mx.array]]]", "docstring": "", "summary": "Function `_normalize_position_inputs` calls `logger.debug`; has 2 explicit return paths.", "implementation": "Function `_normalize_position_inputs` calls `logger.debug`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 53, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L53-L65", "decorators": [], "calls": [ "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(position_ids, position_embeddings)", "(None, None)" ], "awaits": false, "yields": false }, { "name": "_position_ids_for_offset", "qualname": "_position_ids_for_offset", "full_name": "vllm_mlx.patches.qwen3_5_mllm._position_ids_for_offset", "kind": "function", "signature": "def _position_ids_for_offset(offset: int, length: int) -> mx.array", "parameters": [ { "name": "offset", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "length", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "", "summary": "Function `_position_ids_for_offset` calls `mx.arange`, `mx.expand_dims`, `mx.tile`; returns `mx.tile(position_ids, (3, 1, 1))`.", "implementation": "Function `_position_ids_for_offset` calls `mx.arange`, `mx.expand_dims`, `mx.tile`; returns `mx.tile(position_ids, (3, 1, 1))`.", "documented": false, "public": false, "addressable": true, "line": 68, "end_line": 71, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L68-L71", "decorators": [], "calls": [ "mx.arange", "mx.expand_dims", "mx.tile" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.tile(position_ids, (3, 1, 1))" ], "awaits": false, "yields": false }, { "name": "_kv_seq_len", "qualname": "_kv_seq_len", "full_name": "vllm_mlx.patches.qwen3_5_mllm._kv_seq_len", "kind": "function", "signature": "def _kv_seq_len(keys: mx.array, cache: Optional[Any], offset: int) -> int", "parameters": [ { "name": "keys", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "offset", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "", "summary": "Function `_kv_seq_len` returns `length + offset + 1 if cache is not None else length`.", "implementation": "Function `_kv_seq_len` returns `length + offset + 1 if cache is not None else length`.", "documented": false, "public": false, "addressable": true, "line": 74, "end_line": 76, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L74-L76", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "length + offset + 1 if cache is not None else length" ], "awaits": false, "yields": false }, { "name": "_apply_rotary", "qualname": "_apply_rotary", "full_name": "vllm_mlx.patches.qwen3_5_mllm._apply_rotary", "kind": "function", "signature": "def _apply_rotary(attention, queries: mx.array, keys: mx.array, values: mx.array, position_ids: mx.array, position_embeddings: Optional[tuple[mx.array, mx.array]], apply_multimodal_rotary_pos_emb) -> tuple[mx.array, mx.array]", "parameters": [ { "name": "attention", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "queries", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "keys", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "values", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "position_ids", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "position_embeddings", "kind": "positional or keyword", "annotation": "Optional[tuple[mx.array, mx.array]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "apply_multimodal_rotary_pos_emb", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[mx.array, mx.array]", "docstring": "", "summary": "Function `_apply_rotary` calls `apply_multimodal_rotary_pos_emb`, `hasattr`, `attention.rotary_emb.apply_rotary`, `attention.rotary_emb`; has 2 explicit return paths.", "implementation": "Function `_apply_rotary` calls `apply_multimodal_rotary_pos_emb`, `hasattr`, `attention.rotary_emb.apply_rotary`, `attention.rotary_emb`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 79, "end_line": 101, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L79-L101", "decorators": [], "calls": [ "apply_multimodal_rotary_pos_emb", "hasattr", "attention.rotary_emb.apply_rotary", "attention.rotary_emb" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "apply_multimodal_rotary_pos_emb(queries, keys, cos, sin)", "attention.rotary_emb.apply_rotary(queries, keys, position_ids, unsqueeze_dim=1)" ], "awaits": false, "yields": false }, { "name": "_slice_attention_mask", "qualname": "_slice_attention_mask", "full_name": "vllm_mlx.patches.qwen3_5_mllm._slice_attention_mask", "kind": "function", "signature": "def _slice_attention_mask(mask: Optional[mx.array], cache: Optional[Any], kv_seq_len: int, length: int) -> Optional[mx.array]", "parameters": [ { "name": "mask", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "kv_seq_len", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "length", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[mx.array]", "docstring": "", "summary": "Function `_slice_attention_mask` calls `isinstance`, `hasattr`, `int`, `kv_seq_len.max().item`; has 2 explicit return paths.", "implementation": "Function `_slice_attention_mask` calls `isinstance`, `hasattr`, `int`, `kv_seq_len.max().item`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 104, "end_line": 116, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L104-L116", "decorators": [], "calls": [ "isinstance", "hasattr", "int", "kv_seq_len.max().item", "kv_seq_len.max" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mask", "mask[..., :int(kv_seq_len)]" ], "awaits": false, "yields": false }, { "name": "_maybe_target_verify_attention", "qualname": "_maybe_target_verify_attention", "full_name": "vllm_mlx.patches.qwen3_5_mllm._maybe_target_verify_attention", "kind": "function", "signature": "def _maybe_target_verify_attention(queries: mx.array, keys: mx.array, values: mx.array, *, cache: Optional[Any], mask: Optional[mx.array], scale: float, target_verify: bool, length: int, left_padded_decode: bool, target_verify_left_padded_attention) -> Optional[mx.array]", "parameters": [ { "name": "queries", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "keys", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "values", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "keyword-only", "annotation": "Optional[Any]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "mask", "kind": "keyword-only", "annotation": "Optional[mx.array]", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "scale", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "target_verify", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "length", "kind": "keyword-only", "annotation": "int", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "left_padded_decode", "kind": "keyword-only", "annotation": "bool", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "target_verify_left_padded_attention", "kind": "keyword-only", "annotation": "not annotated", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "Optional[mx.array]", "docstring": "", "summary": "Function `_maybe_target_verify_attention` calls `target_verify_left_padded_attention`; has 2 explicit return paths.", "implementation": "Function `_maybe_target_verify_attention` calls `target_verify_left_padded_attention`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 119, "end_line": 141, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L119-L141", "decorators": [], "calls": [ "target_verify_left_padded_attention" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "target_verify_left_padded_attention(queries, keys, values, cache=cache, scale=scale, mask=mask)" ], "awaits": false, "yields": false }, { "name": "patch_qwen35_attention_for_batching", "qualname": "patch_qwen35_attention_for_batching", "full_name": "vllm_mlx.patches.qwen3_5_mllm.patch_qwen35_attention_for_batching", "kind": "function", "signature": "def patch_qwen35_attention_for_batching() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Monkey-patch Qwen3_5Attention.__call__ to handle BatchKVCache.\n\nReturns True if patch was applied, False if mlx-vlm is not installed\nor Qwen3.5 module not available.", "summary": "Monkey-patch Qwen3_5Attention.__call__ to handle BatchKVCache.", "implementation": "Function `patch_qwen35_attention_for_batching` calls `importlib.import_module`, `logger.debug`, `getattr`, `setattr`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 144, "end_line": 266, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L144-L266", "decorators": [], "calls": [ "importlib.import_module", "logger.debug", "getattr", "setattr", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_patched_call", "qualname": "patch_qwen35_attention_for_batching._patched_call", "full_name": "vllm_mlx.patches.qwen3_5_mllm.patch_qwen35_attention_for_batching._patched_call", "kind": "nested function", "signature": "def _patched_call(self, x: mx.array, mask: Optional[mx.array]=None, cache: Optional[Any]=None, position_ids: Optional[mx.array]=None, position_embeddings: Optional[tuple[mx.array, mx.array]]=None, target_verify: bool=False) -> mx.array", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "mask", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "cache", "kind": "positional or keyword", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "position_ids", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "position_embeddings", "kind": "positional or keyword", "annotation": "Optional[tuple[mx.array, mx.array]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "target_verify", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "mx.array", "docstring": "", "summary": "Nested Function `patch_qwen35_attention_for_batching._patched_call` calls `target_verify_linears`, `mx.split`, `q_proj_output.reshape`, `gate.reshape`; returns `self.o_proj(output * mx.sigmoid(gate))`.", "implementation": "Nested Function `patch_qwen35_attention_for_batching._patched_call` calls `target_verify_linears`, `mx.split`, `q_proj_output.reshape`, `gate.reshape`; returns `self.o_proj(output * mx.sigmoid(gate))`.", "documented": false, "public": false, "addressable": false, "line": 174, "end_line": 261, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mllm.py#L174-L261", "decorators": [], "calls": [ "target_verify_linears", "mx.split", "q_proj_output.reshape", "gate.reshape", "self.q_norm(queries).transpose", "self.q_norm", "self.k_norm(keys.reshape(B, L, self.num_key_value_heads, -1)).transpose", "self.k_norm", "keys.reshape", "values.reshape(B, L, self.num_key_value_heads, -1).transpose", "values.reshape", "_cache_offset_to_int", "_normalize_position_inputs", "_position_ids_for_offset", "_kv_seq_len", "_apply_rotary", "_slice_attention_mask", "cache.update_and_fetch", "isinstance", "_maybe_target_verify_attention", "scaled_dot_product_attention", "output.transpose(0, 2, 1, 3).reshape", "output.transpose", "self.o_proj", "mx.sigmoid" ], "state_reads": [ "self.q_proj", "self.k_proj", "self.v_proj", "self.num_attention_heads", "self.q_norm", "self.k_norm", "self.num_key_value_heads", "self.scale", "self.o_proj" ], "state_writes": [], "raises": [], "return_expressions": [ "self.o_proj(output * mx.sigmoid(gate))" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.patches.qwen3_5_mtp", "path": "vllm_mlx/patches/qwen3_5_mtp.py", "page_path": "reference/api/vllm_mlx/patches/qwen3_5_mtp.md", "docstring": "Runtime MTP (Multi-Token Prediction) support for Qwen3.5 models.\n\nQwen3.5 models may include a built-in MTP head that predicts token n+2\nfrom hidden states + token n+1. MTP weights are added to the quantized\nMLX model via scripts/add_mtp_weights_qwen35.py.\n\nSince mlx_lm's qwen3_5.py does NOT define MTP module/methods, this\nmodule provides:\n - inject_mtp_support(): dynamically creates MTP module, loads weights,\n and monkey-patches the model class with return_hidden, mtp_forward,\n and make_mtp_cache\n - validate_mtp_support(): checks whether a loaded model has working MTP\n\nSupports both Dense (27B) and MoE (122B-A10B, 35B-A3B) architectures.\n\nThe actual MTP scheduling logic lives in:\n - vllm_mlx/scheduler.py (_install_mtp, _mtp_step, _mtp_next)", "summary": "Runtime MTP (Multi-Token Prediction) support for Qwen3.5 models.", "line_count": 512, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L1-L512", "members": [ "logger", "_MTP_KEY_PREFIXES", "_strip_mtp_key_prefix", "_QWEN_MTP_RMSNORM_WEIGHT_SUFFIXES", "_QWEN_MTP_HIDDEN_STATE_MODES", "_resolve_qwen_mtp_hidden_state_mode", "_select_qwen_mtp_hidden_state", "_is_qwen_mtp_rmsnorm_weight", "_apply_qwen_mtp_rmsnorm_offset_fixups", "_fixup_moe_mtp", "inject_mtp_support", "validate_mtp_support" ], "symbols": [ { "name": "_strip_mtp_key_prefix", "qualname": "_strip_mtp_key_prefix", "full_name": "vllm_mlx.patches.qwen3_5_mtp._strip_mtp_key_prefix", "kind": "function", "signature": "def _strip_mtp_key_prefix(key: str) -> str | None", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "Return an MTP-relative key for supported standalone shard layouts.", "summary": "Return an MTP-relative key for supported standalone shard layouts.", "implementation": "Function `_strip_mtp_key_prefix` calls `key.startswith`, `key.removeprefix`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 31, "end_line": 36, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L31-L36", "decorators": [], "calls": [ "key.startswith", "key.removeprefix" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "key.removeprefix(prefix)", "None" ], "awaits": false, "yields": false }, { "name": "_resolve_qwen_mtp_hidden_state_mode", "qualname": "_resolve_qwen_mtp_hidden_state_mode", "full_name": "vllm_mlx.patches.qwen3_5_mtp._resolve_qwen_mtp_hidden_state_mode", "kind": "function", "signature": "def _resolve_qwen_mtp_hidden_state_mode(config: dict) -> str", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Resolve the checkpoint's MTP hidden-state contract safely.", "summary": "Resolve the checkpoint's MTP hidden-state contract safely.", "implementation": "Function `_resolve_qwen_mtp_hidden_state_mode` calls `config.get`, `text_config.get`, `isinstance`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 52, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L52-L65", "decorators": [], "calls": [ "config.get", "text_config.get", "isinstance", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'post_norm'", "mode" ], "awaits": false, "yields": false }, { "name": "_select_qwen_mtp_hidden_state", "qualname": "_select_qwen_mtp_hidden_state", "full_name": "vllm_mlx.patches.qwen3_5_mtp._select_qwen_mtp_hidden_state", "kind": "function", "signature": "def _select_qwen_mtp_hidden_state(mode: str, hidden_states, normed)", "parameters": [ { "name": "mode", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "hidden_states", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "normed", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Select the representation expected by the checkpoint's MTP head.", "summary": "Select the representation expected by the checkpoint's MTP head.", "implementation": "Function `_select_qwen_mtp_hidden_state` returns `hidden_states if mode == 'pre_norm' else normed`.", "documented": true, "public": false, "addressable": true, "line": 68, "end_line": 70, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L68-L70", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hidden_states if mode == 'pre_norm' else normed" ], "awaits": false, "yields": false }, { "name": "_is_qwen_mtp_rmsnorm_weight", "qualname": "_is_qwen_mtp_rmsnorm_weight", "full_name": "vllm_mlx.patches.qwen3_5_mtp._is_qwen_mtp_rmsnorm_weight", "kind": "function", "signature": "def _is_qwen_mtp_rmsnorm_weight(key: str, weight) -> bool", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weight", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True for MTP RMSNorm weights that use Qwen's offset convention.", "summary": "Return True for MTP RMSNorm weights that use Qwen's offset convention.", "implementation": "Function `_is_qwen_mtp_rmsnorm_weight` calls `any`, `key.endswith`; returns `weight.ndim == 1 and any((key.endswith(suffix) for suffix in _QWEN_MTP_RMSNORM_WEIGHT_SUFFIXES))`.", "documented": true, "public": false, "addressable": true, "line": 73, "end_line": 77, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L73-L77", "decorators": [], "calls": [ "any", "key.endswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "weight.ndim == 1 and any((key.endswith(suffix) for suffix in _QWEN_MTP_RMSNORM_WEIGHT_SUFFIXES))" ], "awaits": false, "yields": false }, { "name": "_apply_qwen_mtp_rmsnorm_offset_fixups", "qualname": "_apply_qwen_mtp_rmsnorm_offset_fixups", "full_name": "vllm_mlx.patches.qwen3_5_mtp._apply_qwen_mtp_rmsnorm_offset_fixups", "kind": "function", "signature": "def _apply_qwen_mtp_rmsnorm_offset_fixups(mtp_weights: dict) -> int", "parameters": [ { "name": "mtp_weights", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Apply Qwen raw-offset RMSNorm fixups without double-shifting MLX weights.", "summary": "Apply Qwen raw-offset RMSNorm fixups without double-shifting MLX weights.", "implementation": "Function `_apply_qwen_mtp_rmsnorm_offset_fixups` calls `list`, `mtp_weights.items`, `_is_qwen_mtp_rmsnorm_weight`, `weight.mean().item`; returns `norm_fixup_count`.", "documented": true, "public": false, "addressable": true, "line": 80, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L80-L90", "decorators": [], "calls": [ "list", "mtp_weights.items", "_is_qwen_mtp_rmsnorm_weight", "weight.mean().item", "weight.mean" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "norm_fixup_count" ], "awaits": false, "yields": false }, { "name": "_fixup_moe_mtp", "qualname": "_fixup_moe_mtp", "full_name": "vllm_mlx.patches.qwen3_5_mtp._fixup_moe_mtp", "kind": "function", "signature": "def _fixup_moe_mtp(mtp, inner_model, loaded_keys: set, mx) -> None", "parameters": [ { "name": "mtp", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "inner_model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "loaded_keys", "kind": "positional or keyword", "annotation": "set", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "mx", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Fix missing weights in MoE MTP module.\n\nMoE MTP checkpoints (122B, 35B) only contain: fc, q_proj, o_proj,\nshared_expert.*, and per-expert weights. Missing:\n- k_proj, v_proj → zero out (attention becomes no-op)\n- gate, shared_expert_gate → copy from main model's last full-attn layer\n- norms → already at identity (weight=1.0), no action needed", "summary": "Fix missing weights in MoE MTP module.", "implementation": "Function `_fixup_moe_mtp` calls `reversed`, `logger.warning`, `getattr`, `mlx.utils.tree_flatten`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 93, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L93-L157", "decorators": [], "calls": [ "reversed", "logger.warning", "getattr", "mlx.utils.tree_flatten", "src.parameters", "dst.load_weights", "mx.eval", "dst.parameters", "logger.info", "hasattr", "mx.zeros_like", "proj.parameters" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "inject_mtp_support", "qualname": "inject_mtp_support", "full_name": "vllm_mlx.patches.qwen3_5_mtp.inject_mtp_support", "kind": "function", "signature": "def inject_mtp_support(model: Any, model_path, config: dict) -> bool", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "A model loaded via mlx_lm (strict=False, MTP weights ignored)" }, { "name": "model_path", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Path to model directory (contains model-mtp.safetensors)" }, { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Parsed config.json dict" } ], "return_annotation": "bool", "docstring": "Inject MTP module into a loaded Qwen3.5 model.\n\nmlx_lm's qwen3_5.py does not define MTP layers, so we:\n1. Create MTP module matching the weight structure\n2. Quantize it to match the base model\n3. Load MTP weights from model-mtp.safetensors\n4. Monkey-patch Model with return_hidden, mtp_forward, make_mtp_cache\n\nArgs:\n model: A model loaded via mlx_lm (strict=False, MTP weights ignored)\n model_path: Path to model directory (contains model-mtp.safetensors)\n config: Parsed config.json dict\n\nReturns:\n True if MTP was successfully injected, False otherwise.", "summary": "Inject MTP module into a loaded Qwen3.5 model.", "implementation": "Function `inject_mtp_support` calls `config.get`, `_resolve_qwen_mtp_hidden_state_mode`, `text_config.get`, `logger.info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 160, "end_line": 447, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L160-L447", "decorators": [], "calls": [ "config.get", "_resolve_qwen_mtp_hidden_state_mode", "text_config.get", "logger.info", "Path", "mtp_file.exists", "logger.warning", "hasattr", "isinstance", "TextModelArgs.from_dict", "getattr", "_MTPModule", "quant_config.get", "mx.load", "str", "raw.items", "_strip_mtp_key_prefix", "set", "sorted", "raw_mtp.keys", "key.endswith", "key.replace", "mx.dequantize", "processed.update", "processed.add", "list", "mtp_weights.keys", "mtp_weights.pop", "_apply_qwen_mtp_rmsnorm_offset_fixups", "mtp.load_weights", "mtp_weights.items", "mx.eval", "mtp.parameters", "sum", "k.endswith", "any", "_fixup_moe_mtp" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_MTPModule", "qualname": "inject_mtp_support._MTPModule", "full_name": "vllm_mlx.patches.qwen3_5_mtp.inject_mtp_support._MTPModule", "kind": "nested class", "signature": "class _MTPModule(nn.Module)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_layers", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_MTPModule", "docstring": "", "summary": "Nested Class `inject_mtp_support._MTPModule` derives from `nn.Module` and declares 1 direct member(s).", "implementation": "Nested Class `inject_mtp_support._MTPModule` derives from `nn.Module` and declares 1 direct member(s).", "documented": false, "public": false, "addressable": false, "line": 239, "end_line": 252, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L239-L252", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "inject_mtp_support._MTPModule.__init__", "full_name": "vllm_mlx.patches.qwen3_5_mtp.inject_mtp_support._MTPModule.__init__", "kind": "nested function", "signature": "def __init__(self, args, n_layers)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_layers", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `inject_mtp_support._MTPModule.__init__` updates `self.pre_fc_norm_hidden`, `self.pre_fc_norm_embedding`, `self.fc`, `self.layers`; calls `super().__init__`, `super`, `nn.RMSNorm`, `nn.Linear`.", "implementation": "Nested Function `inject_mtp_support._MTPModule.__init__` updates `self.pre_fc_norm_hidden`, `self.pre_fc_norm_embedding`, `self.fc`, `self.layers`; calls `super().__init__`, `super`, `nn.RMSNorm`, `nn.Linear`.", "documented": false, "public": false, "addressable": false, "line": 240, "end_line": 252, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L240-L252", "decorators": [], "calls": [ "super().__init__", "super", "nn.RMSNorm", "nn.Linear", "DecoderLayer", "range" ], "state_reads": [], "state_writes": [ "self.pre_fc_norm_hidden", "self.pre_fc_norm_embedding", "self.fc", "self.layers", "self.norm" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_Qwen3_5MTP", "qualname": "inject_mtp_support._Qwen3_5MTP", "full_name": "vllm_mlx.patches.qwen3_5_mtp.inject_mtp_support._Qwen3_5MTP", "kind": "nested class", "signature": "class _Qwen3_5MTP(original_class)", "parameters": [], "return_annotation": "_Qwen3_5MTP", "docstring": "Qwen3.5 with MTP support (injected at runtime).", "summary": "Qwen3.5 with MTP support (injected at runtime).", "implementation": "Nested Class `inject_mtp_support._Qwen3_5MTP` derives from `original_class` and declares 3 direct member(s).", "documented": true, "public": false, "addressable": false, "line": 368, "end_line": 438, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L368-L438", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "inject_mtp_support._Qwen3_5MTP.__call__", "full_name": "vllm_mlx.patches.qwen3_5_mtp.inject_mtp_support._Qwen3_5MTP.__call__", "kind": "nested function", "signature": "def __call__(self, inputs, cache=None, return_hidden: bool=False, input_embeddings=None, **kwargs)", "parameters": [ { "name": "inputs", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "return_hidden", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." }, { "name": "input_embeddings", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `inject_mtp_support._Qwen3_5MTP.__call__` calls `inner.embed_tokens`, `len`, `create_attention_mask`, `create_ssm_mask`; has 2 explicit return paths.", "implementation": "Nested Function `inject_mtp_support._Qwen3_5MTP.__call__` calls `inner.embed_tokens`, `len`, `create_attention_mask`, `create_ssm_mask`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 371, "end_line": 408, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L371-L408", "decorators": [], "calls": [ "inner.embed_tokens", "len", "create_attention_mask", "create_ssm_mask", "zip", "layer", "inner.norm", "inner.embed_tokens.as_linear", "self.lm_head", "_select_qwen_mtp_hidden_state" ], "state_reads": [ "self.model", "self.args.tie_word_embeddings", "self.args", "self.lm_head" ], "state_writes": [], "raises": [], "return_expressions": [ "(out, _select_qwen_mtp_hidden_state(hidden_state_mode, hidden_states, normed))", "out" ], "awaits": false, "yields": false }, { "name": "mtp_forward", "qualname": "inject_mtp_support._Qwen3_5MTP.mtp_forward", "full_name": "vllm_mlx.patches.qwen3_5_mtp.inject_mtp_support._Qwen3_5MTP.mtp_forward", "kind": "nested function", "signature": "def mtp_forward(self, hidden_states, next_token_ids, cache=None, mtp_cache=None)", "parameters": [ { "name": "hidden_states", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "next_token_ids", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "mtp_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Run MTP head: predict token n+2 from hidden states + token n+1.", "summary": "Run MTP head: predict token n+2 from hidden states + token n+1.", "implementation": "Nested Function `inject_mtp_support._Qwen3_5MTP.mtp_forward` calls `self.model.embed_tokens`, `self.mtp.pre_fc_norm_embedding`, `self.mtp.pre_fc_norm_hidden`, `self.mtp.fc`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 410, "end_line": 432, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L410-L432", "decorators": [], "calls": [ "self.model.embed_tokens", "self.mtp.pre_fc_norm_embedding", "self.mtp.pre_fc_norm_hidden", "self.mtp.fc", "mx.concatenate", "create_attention_mask", "layer", "self.mtp.norm", "self.model.embed_tokens.as_linear", "self.lm_head" ], "state_reads": [ "self.model.embed_tokens", "self.model", "self.mtp.pre_fc_norm_embedding", "self.mtp", "self.mtp.pre_fc_norm_hidden", "self.mtp.fc", "self.mtp.layers", "self.mtp.norm", "self.args.tie_word_embeddings", "self.args", "self.model.embed_tokens.as_linear", "self.lm_head" ], "state_writes": [], "raises": [], "return_expressions": [ "self.model.embed_tokens.as_linear(x)", "self.lm_head(x)" ], "awaits": false, "yields": false }, { "name": "make_mtp_cache", "qualname": "inject_mtp_support._Qwen3_5MTP.make_mtp_cache", "full_name": "vllm_mlx.patches.qwen3_5_mtp.inject_mtp_support._Qwen3_5MTP.make_mtp_cache", "kind": "nested function", "signature": "def make_mtp_cache(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Create KV cache for MTP layers.", "summary": "Create KV cache for MTP layers.", "implementation": "Nested Function `inject_mtp_support._Qwen3_5MTP.make_mtp_cache` calls `KVCache`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 434, "end_line": 438, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L434-L438", "decorators": [], "calls": [ "KVCache" ], "state_reads": [ "self.mtp", "self.mtp.layers" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "[KVCache() for _ in self.mtp.layers]" ], "awaits": false, "yields": false }, { "name": "validate_mtp_support", "qualname": "validate_mtp_support", "full_name": "vllm_mlx.patches.qwen3_5_mtp.validate_mtp_support", "kind": "function", "signature": "def validate_mtp_support(model: Any) -> bool", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "A model loaded via mlx_lm.load()" } ], "return_annotation": "bool", "docstring": "Validate that a loaded model has working MTP support.\n\nChecks:\n1. model.mtp exists and is not None\n2. model.mtp has layers with loaded weights\n3. model has return_hidden support in __call__\n4. model has mtp_forward method\n5. model has make_mtp_cache method\n\nArgs:\n model: A model loaded via mlx_lm.load()\n\nReturns:\n True if MTP is fully functional, False otherwise.", "summary": "Validate that a loaded model has working MTP support.", "implementation": "Function `validate_mtp_support` calls `hasattr`, `getattr`, `logger.warning`, `inspect.signature`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 450, "end_line": 512, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_5_mtp.py#L450-L512", "decorators": [], "calls": [ "hasattr", "getattr", "logger.warning", "inspect.signature", "type", "callable", "logger.info", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.patches.qwen3_next_mtp", "path": "vllm_mlx/patches/qwen3_next_mtp.py", "page_path": "reference/api/vllm_mlx/patches/qwen3_next_mtp.md", "docstring": "Runtime MTP (Multi-Token Prediction) support for Qwen3-Next models.\n\nQwen3-Next models may include a built-in MTP head that predicts token n+2\nfrom hidden states + token n+1. MTP weights are added to the quantized\nMLX model via scripts/add_mtp_weights.py.\n\nSince mlx_lm's qwen3_next.py does NOT define MTP module/methods, this\nmodule provides:\n - inject_mtp_support(): dynamically creates MTP module, loads weights,\n and monkey-patches the model class with return_hidden, mtp_forward,\n and make_mtp_cache\n - validate_mtp_support(): checks whether a loaded model has working MTP\n\nThe actual MTP scheduling logic lives in:\n - vllm_mlx/scheduler.py (_install_mtp, _mtp_step, _mtp_next)", "summary": "Runtime MTP (Multi-Token Prediction) support for Qwen3-Next models.", "line_count": 261, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L1-L261", "members": [ "logger", "inject_mtp_support", "validate_mtp_support" ], "symbols": [ { "name": "inject_mtp_support", "qualname": "inject_mtp_support", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support", "kind": "function", "signature": "def inject_mtp_support(model: Any, model_path, config: dict) -> bool", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "A model loaded via mlx_lm (strict=False, MTP weights ignored)" }, { "name": "model_path", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Path to model directory (contains model-mtp.safetensors)" }, { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Parsed config.json dict" } ], "return_annotation": "bool", "docstring": "Inject MTP module into a loaded Qwen3-Next model.\n\nmlx_lm's qwen3_next.py does not define MTP layers, so we:\n1. Create MTP module matching the weight structure\n2. Quantize it to match the base model\n3. Load MTP weights from model-mtp.safetensors\n4. Monkey-patch Model with return_hidden, mtp_forward, make_mtp_cache\n\nArgs:\n model: A model loaded via mlx_lm (strict=False, MTP weights ignored)\n model_path: Path to model directory (contains model-mtp.safetensors)\n config: Parsed config.json dict\n\nReturns:\n True if MTP was successfully injected, False otherwise.", "summary": "Inject MTP module into a loaded Qwen3-Next model.", "implementation": "Function `inject_mtp_support` calls `config.get`, `logger.info`, `Path`, `mtp_file.exists`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 27, "end_line": 181, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L27-L181", "decorators": [], "calls": [ "config.get", "logger.info", "Path", "mtp_file.exists", "logger.warning", "_MTPModule", "quant_config.get", "nn.quantize", "mx.load", "str", "k.removeprefix", "raw.items", "k.startswith", "mtp.load_weights", "list", "mtp_weights.items", "mx.eval", "mtp.parameters", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_MTPModule", "qualname": "inject_mtp_support._MTPModule", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support._MTPModule", "kind": "nested class", "signature": "class _MTPModule(nn.Module)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_layers", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_MTPModule", "docstring": "", "summary": "Nested Class `inject_mtp_support._MTPModule` derives from `nn.Module` and declares 1 direct member(s).", "implementation": "Nested Class `inject_mtp_support._MTPModule` derives from `nn.Module` and declares 1 direct member(s).", "documented": false, "public": false, "addressable": false, "line": 68, "end_line": 83, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L68-L83", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "inject_mtp_support._MTPModule.__init__", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support._MTPModule.__init__", "kind": "nested function", "signature": "def __init__(self, args, n_layers)", "parameters": [ { "name": "args", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_layers", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `inject_mtp_support._MTPModule.__init__` updates `self.pre_fc_norm_hidden`, `self.pre_fc_norm_embedding`, `self.fc`, `self.layers`; calls `super().__init__`, `super`, `nn.RMSNorm`, `nn.Linear`.", "implementation": "Nested Function `inject_mtp_support._MTPModule.__init__` updates `self.pre_fc_norm_hidden`, `self.pre_fc_norm_embedding`, `self.fc`, `self.layers`; calls `super().__init__`, `super`, `nn.RMSNorm`, `nn.Linear`.", "documented": false, "public": false, "addressable": false, "line": 69, "end_line": 83, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L69-L83", "decorators": [], "calls": [ "super().__init__", "super", "nn.RMSNorm", "nn.Linear", "Qwen3NextDecoderLayer", "range" ], "state_reads": [], "state_writes": [ "self.pre_fc_norm_hidden", "self.pre_fc_norm_embedding", "self.fc", "self.layers", "self.norm" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_mtp_quant_pred", "qualname": "inject_mtp_support._mtp_quant_pred", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support._mtp_quant_pred", "kind": "nested function", "signature": "def _mtp_quant_pred(path, module)", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "module", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `inject_mtp_support._mtp_quant_pred` calls `isinstance`, `path.endswith`; has 3 explicit return paths.", "implementation": "Nested Function `inject_mtp_support._mtp_quant_pred` calls `isinstance`, `path.endswith`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 93, "end_line": 103, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L93-L103", "decorators": [], "calls": [ "isinstance", "path.endswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "{'group_size': 64, 'bits': 8}", "True" ], "awaits": false, "yields": false }, { "name": "_Qwen3NextMTP", "qualname": "inject_mtp_support._Qwen3NextMTP", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support._Qwen3NextMTP", "kind": "nested class", "signature": "class _Qwen3NextMTP(original_class)", "parameters": [], "return_annotation": "_Qwen3NextMTP", "docstring": "Qwen3-Next with MTP support (injected at runtime).", "summary": "Qwen3-Next with MTP support (injected at runtime).", "implementation": "Nested Class `inject_mtp_support._Qwen3NextMTP` derives from `original_class` and declares 3 direct member(s).", "documented": true, "public": false, "addressable": false, "line": 125, "end_line": 177, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L125-L177", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "inject_mtp_support._Qwen3NextMTP.__call__", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support._Qwen3NextMTP.__call__", "kind": "nested function", "signature": "def __call__(self, inputs, cache=None, return_hidden: bool=False)", "parameters": [ { "name": "inputs", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "return_hidden", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `inject_mtp_support._Qwen3NextMTP.__call__` calls `inner.embed_tokens`, `len`, `create_attention_mask`, `create_ssm_mask`; has 2 explicit return paths.", "implementation": "Nested Function `inject_mtp_support._Qwen3NextMTP.__call__` calls `inner.embed_tokens`, `len`, `create_attention_mask`, `create_ssm_mask`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 128, "end_line": 150, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L128-L150", "decorators": [], "calls": [ "inner.embed_tokens", "len", "create_attention_mask", "create_ssm_mask", "zip", "layer", "inner.norm", "inner.embed_tokens.as_linear", "self.lm_head" ], "state_reads": [ "self.model", "self.args.tie_word_embeddings", "self.args", "self.lm_head" ], "state_writes": [], "raises": [], "return_expressions": [ "(out, hidden_states)", "out" ], "awaits": false, "yields": false }, { "name": "mtp_forward", "qualname": "inject_mtp_support._Qwen3NextMTP.mtp_forward", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support._Qwen3NextMTP.mtp_forward", "kind": "nested function", "signature": "def mtp_forward(self, hidden_states, next_token_ids, cache=None, mtp_cache=None)", "parameters": [ { "name": "hidden_states", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "next_token_ids", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "mtp_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Run MTP head: predict token n+2 from hidden states + token n+1.", "summary": "Run MTP head: predict token n+2 from hidden states + token n+1.", "implementation": "Nested Function `inject_mtp_support._Qwen3NextMTP.mtp_forward` calls `self.model.embed_tokens`, `self.mtp.pre_fc_norm_hidden`, `self.mtp.pre_fc_norm_embedding`, `self.mtp.fc`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 152, "end_line": 171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L152-L171", "decorators": [], "calls": [ "self.model.embed_tokens", "self.mtp.pre_fc_norm_hidden", "self.mtp.pre_fc_norm_embedding", "self.mtp.fc", "mx.concatenate", "create_attention_mask", "layer", "self.mtp.norm", "self.model.embed_tokens.as_linear", "self.lm_head" ], "state_reads": [ "self.model.embed_tokens", "self.model", "self.mtp.pre_fc_norm_hidden", "self.mtp", "self.mtp.pre_fc_norm_embedding", "self.mtp.fc", "self.mtp.layers", "self.mtp.norm", "self.args.tie_word_embeddings", "self.args", "self.model.embed_tokens.as_linear", "self.lm_head" ], "state_writes": [], "raises": [], "return_expressions": [ "self.model.embed_tokens.as_linear(x)", "self.lm_head(x)" ], "awaits": false, "yields": false }, { "name": "make_mtp_cache", "qualname": "inject_mtp_support._Qwen3NextMTP.make_mtp_cache", "full_name": "vllm_mlx.patches.qwen3_next_mtp.inject_mtp_support._Qwen3NextMTP.make_mtp_cache", "kind": "nested function", "signature": "def make_mtp_cache(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Create KV cache for MTP layers.", "summary": "Create KV cache for MTP layers.", "implementation": "Nested Function `inject_mtp_support._Qwen3NextMTP.make_mtp_cache` calls `KVCache`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 173, "end_line": 177, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L173-L177", "decorators": [], "calls": [ "KVCache" ], "state_reads": [ "self.mtp", "self.mtp.layers" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "[KVCache() for _ in self.mtp.layers]" ], "awaits": false, "yields": false }, { "name": "validate_mtp_support", "qualname": "validate_mtp_support", "full_name": "vllm_mlx.patches.qwen3_next_mtp.validate_mtp_support", "kind": "function", "signature": "def validate_mtp_support(model: Any) -> bool", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "A model loaded via mlx_lm.load()" } ], "return_annotation": "bool", "docstring": "Validate that a loaded model has working MTP support.\n\nChecks:\n1. model.mtp exists and is not None (MTP module instantiated)\n2. model.mtp has layers with loaded weights\n3. model has return_hidden support in __call__\n4. model has mtp_forward method\n5. model has make_mtp_cache method\n\nArgs:\n model: A model loaded via mlx_lm.load()\n\nReturns:\n True if MTP is fully functional, False otherwise.", "summary": "Validate that a loaded model has working MTP support.", "implementation": "Function `validate_mtp_support` calls `getattr`, `logger.warning`, `logger.info`, `inspect.signature`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 184, "end_line": 261, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/patches/qwen3_next_mtp.py#L184-L261", "decorators": [], "calls": [ "getattr", "logger.warning", "logger.info", "inspect.signature", "type", "hasattr", "callable", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.plugin", "path": "vllm_mlx/plugin.py", "page_path": "reference/api/vllm_mlx/plugin.md", "docstring": "vLLM Platform Plugin for MLX.\n\nThis module provides the entry point for vLLM's platform plugin system,\nenabling automatic detection and activation of the MLX platform on\nApple Silicon Macs.", "summary": "vLLM Platform Plugin for MLX.", "line_count": 155, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/plugin.py#L1-L155", "members": [ "logger", "mlx_platform_plugin", "is_mlx_available", "get_mlx_device_info" ], "symbols": [ { "name": "mlx_platform_plugin", "qualname": "mlx_platform_plugin", "full_name": "vllm_mlx.plugin.mlx_platform_plugin", "kind": "function", "signature": "def mlx_platform_plugin() -> str | None", "parameters": [], "return_annotation": "str | None", "docstring": "Platform plugin entry point for vLLM.\n\nThis function is called by vLLM's platform detection system to\ndetermine if the MLX platform should be activated.\n\nReturns:\n str: Fully qualified class name of MLXPlatform if conditions are met\n None: If MLX platform should not be activated", "summary": "Platform plugin entry point for vLLM.", "implementation": "Function `mlx_platform_plugin` calls `logger.debug`, `platform.machine`, `mx.array`, `mx.sum`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 17, "end_line": 70, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/plugin.py#L17-L70", "decorators": [], "calls": [ "logger.debug", "platform.machine", "mx.array", "mx.sum", "mx.default_device", "getattr", "logger.warning", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "'vllm_mlx.vllm_platform.MLXPlatform'" ], "awaits": false, "yields": false }, { "name": "is_mlx_available", "qualname": "is_mlx_available", "full_name": "vllm_mlx.plugin.is_mlx_available", "kind": "function", "signature": "def is_mlx_available() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if MLX platform can be used.\n\nReturns:\n bool: True if MLX is available and working", "summary": "Check if MLX platform can be used.", "implementation": "Function `is_mlx_available` calls `mlx_platform_plugin`; returns `mlx_platform_plugin() is not None`.", "documented": true, "public": true, "addressable": true, "line": 73, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/plugin.py#L73-L80", "decorators": [], "calls": [ "mlx_platform_plugin" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mlx_platform_plugin() is not None" ], "awaits": false, "yields": false }, { "name": "get_mlx_device_info", "qualname": "get_mlx_device_info", "full_name": "vllm_mlx.plugin.get_mlx_device_info", "kind": "function", "signature": "def get_mlx_device_info() -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get information about the MLX device.\n\nReturns:\n dict: Device information including chip name, memory, etc.", "summary": "Get information about the MLX device.", "implementation": "Function `get_mlx_device_info` calls `is_mlx_available`, `subprocess.run`, `result.stdout.strip`, `int`; returns `info`.", "documented": true, "public": true, "addressable": true, "line": 83, "end_line": 155, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/plugin.py#L83-L155", "decorators": [], "calls": [ "is_mlx_available", "subprocess.run", "result.stdout.strip", "int", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "info" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.prefix_cache", "path": "vllm_mlx/prefix_cache.py", "page_path": "reference/api/vllm_mlx/prefix_cache.md", "docstring": "Prefix Cache Manager for vllm-mlx.\n\nWraps mlx-lm's LRUPromptCache to provide prefix caching functionality,\nallowing reuse of computed KV cache for common prompt prefixes.\n\nThis module provides two implementations:\n- PrefixCacheManager: Original trie-based LRU cache (for backward compatibility)\n- BlockAwarePrefixCache: Block-based cache with PagedCacheManager integration", "summary": "Prefix Cache Manager for vllm-mlx.", "line_count": 1039, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L1-L1039", "members": [ "logger", "CacheEntry", "PrefixCacheStats", "PrefixCacheManager", "BlockCacheEntry", "BlockAwarePrefixCache" ], "symbols": [ { "name": "CacheEntry", "qualname": "CacheEntry", "full_name": "vllm_mlx.prefix_cache.CacheEntry", "kind": "class", "signature": "class CacheEntry", "parameters": [ { "name": "prompt_cache", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "count", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "CacheEntry", "docstring": "Entry in the prefix cache.", "summary": "Entry in the prefix cache.", "implementation": "Class `CacheEntry` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 33, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L33-L37", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "PrefixCacheStats", "qualname": "PrefixCacheStats", "full_name": "vllm_mlx.prefix_cache.PrefixCacheStats", "kind": "class", "signature": "class PrefixCacheStats", "parameters": [ { "name": "hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "tokens_saved", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "total_queries", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "evictions", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "PrefixCacheStats", "docstring": "Statistics for prefix cache performance.", "summary": "Statistics for prefix cache performance.", "implementation": "Class `PrefixCacheStats` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 41, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L41-L66", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "hit_rate", "qualname": "PrefixCacheStats.hit_rate", "full_name": "vllm_mlx.prefix_cache.PrefixCacheStats.hit_rate", "kind": "method", "signature": "def hit_rate(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Calculate cache hit rate.", "summary": "Calculate cache hit rate.", "implementation": "Method `PrefixCacheStats.hit_rate` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 51, "end_line": 55, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L51-L55", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.total_queries", "self.hits" ], "state_writes": [], "raises": [], "return_expressions": [ "0.0", "self.hits / self.total_queries" ], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "PrefixCacheStats.to_dict", "full_name": "vllm_mlx.prefix_cache.PrefixCacheStats.to_dict", "kind": "method", "signature": "def to_dict(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Convert stats to dictionary.", "summary": "Convert stats to dictionary.", "implementation": "Method `PrefixCacheStats.to_dict` returns `{'hits': self.hits, 'misses': self.misses, 'hit_rate': self.hit_rate, 'tokens_saved': self.tokens_saved, 'total_queries…`.", "documented": true, "public": true, "addressable": true, "line": 57, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L57-L66", "decorators": [], "calls": [], "state_reads": [ "self.hits", "self.misses", "self.hit_rate", "self.tokens_saved", "self.total_queries", "self.evictions" ], "state_writes": [], "raises": [], "return_expressions": [ "{'hits': self.hits, 'misses': self.misses, 'hit_rate': self.hit_rate, 'tokens_saved': self.tokens_saved, 'total_queries…" ], "awaits": false, "yields": false }, { "name": "PrefixCacheManager", "qualname": "PrefixCacheManager", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager", "kind": "class", "signature": "class PrefixCacheManager", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model (used for cache key identification)" }, { "name": "max_entries", "kind": "positional or keyword", "annotation": "int", "default": "100", "required": false, "description": "Maximum number of cached entries before LRU eviction" } ], "return_annotation": "PrefixCacheManager", "docstring": "Manages prefix caching for vllm-mlx using a trie-based LRU cache.\n\nThis implementation is inspired by mlx-lm's LRUPromptCache but adapted\nfor vllm-mlx's batching architecture.\n\nThe cache stores KV states keyed by token sequences, allowing:\n- Exact match: Full prompt found in cache\n- Shorter match: Partial prefix found, process remaining tokens\n- Longer match: Cached prefix longer than request, trim excess\n\nExample:\n cache_manager = PrefixCacheManager(model, max_entries=100)\n\n # Check for cached prefix\n cache, remaining_tokens = cache_manager.fetch_cache(tokens)\n if cache:\n # Use cached KV, only process remaining_tokens\n pass\n\n # After generation, store cache for reuse\n cache_manager.store_cache(full_tokens, prompt_cache)", "summary": "Manages prefix caching for vllm-mlx using a trie-based LRU cache.", "implementation": "Class `PrefixCacheManager` declares 14 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 355, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L69-L355", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "PrefixCacheManager.__init__", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager.__init__", "kind": "method", "signature": "def __init__(self, model: Any, max_entries: int=100)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model (used for cache key identification)" }, { "name": "max_entries", "kind": "positional or keyword", "annotation": "int", "default": "100", "required": false, "description": "Maximum number of cached entries before LRU eviction" } ], "return_annotation": "not annotated", "docstring": "Initialize the prefix cache manager.\n\nArgs:\n model: The MLX model (used for cache key identification)\n max_entries: Maximum number of cached entries before LRU eviction", "summary": "Initialize the prefix cache manager.", "implementation": "Method `PrefixCacheManager.__init__` updates `self.model`, `self.model_key`, `self.max_size`, `self._cache`; calls `id`, `OrderedDict`, `PrefixCacheStats`.", "documented": true, "public": false, "addressable": true, "line": 94, "end_line": 115, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L94-L115", "decorators": [], "calls": [ "id", "OrderedDict", "PrefixCacheStats" ], "state_reads": [], "state_writes": [ "self.model", "self.model_key", "self.max_size", "self._cache", "self._lru", "self.stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_search", "qualname": "PrefixCacheManager._search", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager._search", "kind": "method", "signature": "def _search(self, tokens: List[int]) -> Tuple[Optional[List[int]], Optional[List[int]], Optional[List[int]], int]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Tuple[Optional[List[int]], Optional[List[int]], Optional[List[int]], int]", "docstring": "Search for cached prefix matching tokens.\n\nReturns:\n Tuple of (exact, shorter, longer, common_prefix_len)\n - exact: Tokens if exact match found\n - shorter: Tokens of shorter cached prefix\n - longer: Tokens of longer cached prefix\n - common_prefix_len: Length of common prefix with longer match", "summary": "Search for cached prefix matching tokens.", "implementation": "Method `PrefixCacheManager._search` calls `enumerate`, `list`, `path.append`, `stack.pop`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 117, "end_line": 164, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L117-L164", "decorators": [], "calls": [ "enumerate", "list", "path.append", "stack.pop", "len", "node.items", "stack.append" ], "state_reads": [ "self.model_key", "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [ "(None, None, None, 0)", "(None, list(path), None, 0)", "(list(tokens), None, None, 0)", "(None, None, node_path, len(tokens))" ], "awaits": false, "yields": false }, { "name": "fetch_cache", "qualname": "PrefixCacheManager.fetch_cache", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager.fetch_cache", "kind": "method", "signature": "def fetch_cache(self, tokens: List[int]) -> Tuple[Optional[List[Any]], List[int]]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Input token sequence" } ], "return_annotation": "Tuple[Optional[List[Any]], List[int]]", "docstring": "Find cached prefix for the given tokens.\n\nArgs:\n tokens: Input token sequence\n\nReturns:\n Tuple of (cache, remaining_tokens)\n - cache: Cached KV state if found, None otherwise\n - remaining_tokens: Tokens that still need processing", "summary": "Find cached prefix for the given tokens.", "implementation": "Method `PrefixCacheManager.fetch_cache` updates `self.stats.total_queries`, `self.stats.hits`, `self.stats.tokens_saved`, `self.stats.misses`; calls `tuple`, `self._search`, `self._get_cache_entry`, `len`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 166, "end_line": 221, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L166-L221", "decorators": [], "calls": [ "tuple", "self._search", "self._get_cache_entry", "len", "self._touch_lru", "self._can_trim_cache", "self._trim_cache", "copy.deepcopy" ], "state_reads": [ "self.stats", "self._search", "self._get_cache_entry", "self._touch_lru", "self._can_trim_cache", "self._trim_cache" ], "state_writes": [ "self.stats.total_queries", "self.stats.hits", "self.stats.tokens_saved", "self.stats.misses" ], "raises": [], "return_expressions": [ "(cache_entry.prompt_cache, [])", "(cache_entry.prompt_cache, remaining)", "(trimmed_cache, [])", "(None, tokens)" ], "awaits": false, "yields": false }, { "name": "store_cache", "qualname": "PrefixCacheManager.store_cache", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager.store_cache", "kind": "method", "signature": "def store_cache(self, tokens: List[int], prompt_cache: List[Any]) -> None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Token sequence that was processed" }, { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "The computed KV cache to store" } ], "return_annotation": "None", "docstring": "Store computed cache for future reuse.\n\nArgs:\n tokens: Token sequence that was processed\n prompt_cache: The computed KV cache to store", "summary": "Store computed cache for future reuse.", "implementation": "Method `PrefixCacheManager.store_cache` calls `tuple`, `self._lru.move_to_end`, `CacheEntry`, `len`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 223, "end_line": 258, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L223-L258", "decorators": [], "calls": [ "tuple", "self._lru.move_to_end", "CacheEntry", "len", "self._evict_lru" ], "state_reads": [ "self.model_key", "self._cache", "self._lru.move_to_end", "self._lru", "self.max_size", "self._evict_lru" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_get_cache_entry", "qualname": "PrefixCacheManager._get_cache_entry", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager._get_cache_entry", "kind": "method", "signature": "def _get_cache_entry(self, tokens: List[int]) -> Optional[CacheEntry]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[CacheEntry]", "docstring": "Get cache entry for given tokens.", "summary": "Get cache entry for given tokens.", "implementation": "Method `PrefixCacheManager._get_cache_entry` calls `current.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 260, "end_line": 271, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L260-L271", "decorators": [], "calls": [ "current.get" ], "state_reads": [ "self.model_key", "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "current.get('cache')" ], "awaits": false, "yields": false }, { "name": "_touch_lru", "qualname": "PrefixCacheManager._touch_lru", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager._touch_lru", "kind": "method", "signature": "def _touch_lru(self, tokens_tuple: tuple) -> None", "parameters": [ { "name": "tokens_tuple", "kind": "positional or keyword", "annotation": "tuple", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Move entry to most-recently-used position — O(1) with OrderedDict.", "summary": "Move entry to most-recently-used position — O(1) with OrderedDict.", "implementation": "Method `PrefixCacheManager._touch_lru` calls `self._lru.move_to_end`.", "documented": true, "public": false, "addressable": true, "line": 273, "end_line": 279, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L273-L279", "decorators": [], "calls": [ "self._lru.move_to_end" ], "state_reads": [ "self.model_key", "self._lru", "self._lru.move_to_end" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_evict_lru", "qualname": "PrefixCacheManager._evict_lru", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager._evict_lru", "kind": "method", "signature": "def _evict_lru(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Evict least recently used entry — O(1) popitem from OrderedDict.", "summary": "Evict least recently used entry — O(1) popitem from OrderedDict.", "implementation": "Method `PrefixCacheManager._evict_lru` updates `self.stats.evictions`; calls `self._lru.popitem`, `self._delete_cache`, `list`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 281, "end_line": 288, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L281-L288", "decorators": [], "calls": [ "self._lru.popitem", "self._delete_cache", "list" ], "state_reads": [ "self._lru", "self._lru.popitem", "self._delete_cache", "self.stats" ], "state_writes": [ "self.stats.evictions" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_delete_cache", "qualname": "PrefixCacheManager._delete_cache", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager._delete_cache", "kind": "method", "signature": "def _delete_cache(self, model_key: Any, tokens: List[int]) -> None", "parameters": [ { "name": "model_key", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Delete cache entry and clean up empty trie branches.", "summary": "Delete cache entry and clean up empty trie branches.", "implementation": "Method `PrefixCacheManager._delete_cache` calls `path.append`, `range`, `len`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 290, "end_line": 314, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L290-L314", "decorators": [], "calls": [ "path.append", "range", "len" ], "state_reads": [ "self._cache" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_can_trim_cache", "qualname": "PrefixCacheManager._can_trim_cache", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager._can_trim_cache", "kind": "method", "signature": "def _can_trim_cache(self, prompt_cache: List[Any]) -> bool", "parameters": [ { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if cache can be trimmed.", "summary": "Check if cache can be trimmed.", "implementation": "Method `PrefixCacheManager._can_trim_cache` calls `hasattr`, `first_cache.is_trimmable`, `logger.debug`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 316, "end_line": 330, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L316-L330", "decorators": [], "calls": [ "hasattr", "first_cache.is_trimmable", "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "trimmable", "hasattr(first_cache, 'trim')" ], "awaits": false, "yields": false }, { "name": "_trim_cache", "qualname": "PrefixCacheManager._trim_cache", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager._trim_cache", "kind": "method", "signature": "def _trim_cache(self, prompt_cache: List[Any], num_tokens: int) -> List[Any]", "parameters": [ { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "List[Any]", "docstring": "Trim cache by removing num_tokens from the end.", "summary": "Trim cache by removing num_tokens from the end.", "implementation": "Method `PrefixCacheManager._trim_cache` calls `hasattr`, `cache.trim`; returns `prompt_cache`.", "documented": true, "public": false, "addressable": true, "line": 332, "end_line": 337, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L332-L337", "decorators": [], "calls": [ "hasattr", "cache.trim" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "prompt_cache" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "PrefixCacheManager.get_stats", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager.get_stats", "kind": "method", "signature": "def get_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get cache statistics.", "summary": "Get cache statistics.", "implementation": "Method `PrefixCacheManager.get_stats` calls `self.stats.to_dict`; returns `self.stats.to_dict()`.", "documented": true, "public": true, "addressable": true, "line": 339, "end_line": 341, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L339-L341", "decorators": [], "calls": [ "self.stats.to_dict" ], "state_reads": [ "self.stats.to_dict", "self.stats" ], "state_writes": [], "raises": [], "return_expressions": [ "self.stats.to_dict()" ], "awaits": false, "yields": false }, { "name": "reset_stats", "qualname": "PrefixCacheManager.reset_stats", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager.reset_stats", "kind": "method", "signature": "def reset_stats(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset statistics.", "summary": "Reset statistics.", "implementation": "Method `PrefixCacheManager.reset_stats` updates `self.stats`; calls `PrefixCacheStats`.", "documented": true, "public": true, "addressable": true, "line": 343, "end_line": 345, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L343-L345", "decorators": [], "calls": [ "PrefixCacheStats" ], "state_reads": [], "state_writes": [ "self.stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "clear", "qualname": "PrefixCacheManager.clear", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear all cached entries.", "summary": "Clear all cached entries.", "implementation": "Method `PrefixCacheManager.clear` calls `self._cache.clear`, `self._lru.clear`, `self.reset_stats`.", "documented": true, "public": true, "addressable": true, "line": 347, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L347-L351", "decorators": [], "calls": [ "self._cache.clear", "self._lru.clear", "self.reset_stats" ], "state_reads": [ "self._cache.clear", "self._cache", "self._lru.clear", "self._lru", "self.reset_stats" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "PrefixCacheManager.__len__", "full_name": "vllm_mlx.prefix_cache.PrefixCacheManager.__len__", "kind": "method", "signature": "def __len__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return number of cached entries.", "summary": "Return number of cached entries.", "implementation": "Method `PrefixCacheManager.__len__` calls `len`; returns `len(self._lru)`.", "documented": true, "public": false, "addressable": true, "line": 353, "end_line": 355, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L353-L355", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._lru" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self._lru)" ], "awaits": false, "yields": false }, { "name": "BlockCacheEntry", "qualname": "BlockCacheEntry", "full_name": "vllm_mlx.prefix_cache.BlockCacheEntry", "kind": "class", "signature": "class BlockCacheEntry", "parameters": [ { "name": "block_table", "kind": "field", "annotation": "BlockTable", "default": "", "required": true, "description": "Required constructor field." }, { "name": "cache_data", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "last_access", "kind": "field", "annotation": "float", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "BlockCacheEntry", "docstring": "Entry mapping a token sequence to cache blocks.", "summary": "Entry mapping a token sequence to cache blocks.", "implementation": "Class `BlockCacheEntry` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 364, "end_line": 369, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L364-L369", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "BlockAwarePrefixCache", "qualname": "BlockAwarePrefixCache", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache", "kind": "class", "signature": "class BlockAwarePrefixCache", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model (used for identification)" }, { "name": "paged_cache_manager", "kind": "positional or keyword", "annotation": "PagedCacheManager", "default": "", "required": true, "description": "The PagedCacheManager instance for block management" } ], "return_annotation": "BlockAwarePrefixCache", "docstring": "Prefix cache that uses PagedCacheManager for block-based storage.\n\nFeatures:\n- Block-level prefix sharing (64 tokens per block)\n- Copy-on-Write for efficient forking\n- Hash-based deduplication across requests\n- Reference counting for memory efficiency\n\nThis is the recommended cache for production use when memory\nefficiency for concurrent requests is important.\n\nExample:\n paged_manager = PagedCacheManager(block_size=64, max_blocks=1000)\n cache = BlockAwarePrefixCache(model, paged_manager)\n\n # Check for cached prefix\n block_table, remaining_tokens = cache.fetch_cache(request_id, tokens)\n\n # After generation, store cache\n cache.store_cache(request_id, tokens, kv_cache_data)\n\n # Clean up when request completes\n cache.release_cache(request_id)", "summary": "Prefix cache that uses PagedCacheManager for block-based storage.", "implementation": "Class `BlockAwarePrefixCache` declares 17 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 372, "end_line": 1039, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L372-L1039", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "BlockAwarePrefixCache.__init__", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.__init__", "kind": "method", "signature": "def __init__(self, model: Any, paged_cache_manager: PagedCacheManager)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model (used for identification)" }, { "name": "paged_cache_manager", "kind": "positional or keyword", "annotation": "PagedCacheManager", "default": "", "required": true, "description": "The PagedCacheManager instance for block management" } ], "return_annotation": "not annotated", "docstring": "Initialize block-aware prefix cache.\n\nArgs:\n model: The MLX model (used for identification)\n paged_cache_manager: The PagedCacheManager instance for block management", "summary": "Initialize block-aware prefix cache.", "implementation": "Method `BlockAwarePrefixCache.__init__` updates `self.model`, `self.model_key`, `self.paged_cache`, `self.block_size`; calls `id`.", "documented": true, "public": false, "addressable": true, "line": 399, "end_line": 426, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L399-L426", "decorators": [], "calls": [ "id" ], "state_reads": [], "state_writes": [ "self.model", "self.model_key", "self.paged_cache", "self.block_size", "self._prefix_index", "self._request_tables", "self._hits", "self._misses", "self._tokens_saved" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "fetch_cache", "qualname": "BlockAwarePrefixCache.fetch_cache", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.fetch_cache", "kind": "method", "signature": "def fetch_cache(self, request_id: str, tokens: List[int]) -> Tuple[Optional[BlockTable], List[int]]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Unique request identifier" }, { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Input token sequence" } ], "return_annotation": "Tuple[Optional[BlockTable], List[int]]", "docstring": "Find cached prefix blocks for the given tokens.\n\nArgs:\n request_id: Unique request identifier\n tokens: Input token sequence\n\nReturns:\n Tuple of (block_table, remaining_tokens)\n - block_table: BlockTable if prefix found, None otherwise\n - remaining_tokens: Tokens that need processing", "summary": "Find cached prefix blocks for the given tokens.", "implementation": "Method `BlockAwarePrefixCache.fetch_cache` updates `self._hits`, `self._tokens_saved`, `self._misses`; calls `self.paged_cache.find_shared_prefix`, `self.paged_cache.create_block_table`, `self.paged_cache.increment_ref`, `self.paged_cache.allocated_blocks.get`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 428, "end_line": 502, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L428-L502", "decorators": [], "calls": [ "self.paged_cache.find_shared_prefix", "self.paged_cache.create_block_table", "self.paged_cache.increment_ref", "self.paged_cache.allocated_blocks.get", "block_table.block_ids.append", "len", "logger.debug", "self._find_best_prefix_match" ], "state_reads": [ "self.paged_cache.find_shared_prefix", "self.paged_cache", "self.paged_cache.create_block_table", "self.paged_cache.increment_ref", "self.paged_cache.allocated_blocks.get", "self.paged_cache.allocated_blocks", "self._find_best_prefix_match" ], "state_writes": [ "self._hits", "self._tokens_saved", "self._misses" ], "raises": [], "return_expressions": [ "(None, tokens)", "(block_table, remaining)" ], "awaits": false, "yields": false }, { "name": "store_cache", "qualname": "BlockAwarePrefixCache.store_cache", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.store_cache", "kind": "method", "signature": "def store_cache(self, request_id: str, tokens: List[int], cache_data: List[Any]) -> Optional[BlockTable]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Unique request identifier" }, { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Token sequence that was processed" }, { "name": "cache_data", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "The computed KV cache to store. Can be: - List of KVCache objects (legacy, stores references) - List of dicts with 'state': (keys, values) tensors (new, stores slices)" } ], "return_annotation": "Optional[BlockTable]", "docstring": "Store computed cache for future reuse.\n\nThis method stores actual tensor data (not references) when cache_data\ncontains extracted states from mlx-lm's KVCache.state property.\n\nArgs:\n request_id: Unique request identifier\n tokens: Token sequence that was processed\n cache_data: The computed KV cache to store. Can be:\n - List of KVCache objects (legacy, stores references)\n - List of dicts with 'state': (keys, values) tensors (new, stores slices)\n\nReturns:\n BlockTable for the stored cache, or None on failure", "summary": "Store computed cache for future reuse.", "implementation": "Method `BlockAwarePrefixCache.store_cache` calls `isinstance`, `len`, `self.paged_cache.get_block_table`, `self.paged_cache.create_block_table`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 504, "end_line": 628, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L504-L628", "decorators": [], "calls": [ "isinstance", "len", "self.paged_cache.get_block_table", "self.paged_cache.create_block_table", "range", "min", "self.paged_cache.find_cached_block", "self.paged_cache.increment_ref", "block_table.block_ids.append", "self.paged_cache.allocate_block", "self.paged_cache.handle_memory_pressure", "logger.warning", "self._extract_block_tensor_slice", "logger.debug", "self.paged_cache.register_block_hash", "self._update_prefix_index", "BlockCacheEntry", "time.time", "sum", "self.paged_cache.allocated_blocks.get" ], "state_reads": [ "self.paged_cache.get_block_table", "self.paged_cache", "self.paged_cache.create_block_table", "self.block_size", "self.paged_cache.find_cached_block", "self.paged_cache.increment_ref", "self.paged_cache.allocate_block", "self.paged_cache.handle_memory_pressure", "self._extract_block_tensor_slice", "self.paged_cache.register_block_hash", "self._update_prefix_index", "self._request_tables", "self.paged_cache.allocated_blocks.get", "self.paged_cache.allocated_blocks" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "block_table" ], "awaits": false, "yields": false }, { "name": "_extract_block_tensor_slice", "qualname": "BlockAwarePrefixCache._extract_block_tensor_slice", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache._extract_block_tensor_slice", "kind": "method", "signature": "def _extract_block_tensor_slice(self, cache_data: List[Dict[str, Any]], start_idx: int, end_idx: int, total_tokens: int) -> Optional[List[Optional[Dict[str, Any]]]]", "parameters": [ { "name": "cache_data", "kind": "positional or keyword", "annotation": "List[Dict[str, Any]]", "default": "", "required": true, "description": "List of extracted layer states" }, { "name": "start_idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Start token index in the sequence" }, { "name": "end_idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "End token index in the sequence" }, { "name": "total_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Total number of tokens covered by cache_data" } ], "return_annotation": "Optional[List[Optional[Dict[str, Any]]]]", "docstring": "Extract per-layer cache data for a single block.\n\nArgs:\n cache_data: List of extracted layer states\n start_idx: Start token index in the sequence\n end_idx: End token index in the sequence\n total_tokens: Total number of tokens covered by cache_data\n\nReturns:\n Per-layer block cache state, or None on failure", "summary": "Extract per-layer cache data for a single block.", "implementation": "Method `BlockAwarePrefixCache._extract_block_tensor_slice` calls `block_slices.append`, `layer_state.get`, `self._cache_state_seq_axis`, `self._slice_concat_cache_state`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 630, "end_line": 702, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L630-L702", "decorators": [], "calls": [ "block_slices.append", "layer_state.get", "self._cache_state_seq_axis", "self._slice_concat_cache_state", "any", "logger.warning" ], "state_reads": [ "self._cache_state_seq_axis", "self._slice_concat_cache_state" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "block_slices if any((entry is not None for entry in block_slices)) else None" ], "awaits": false, "yields": false }, { "name": "_cache_state_seq_axis", "qualname": "BlockAwarePrefixCache._cache_state_seq_axis", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache._cache_state_seq_axis", "kind": "method", "signature": "def _cache_state_seq_axis(self, state: Any) -> Optional[int]", "parameters": [ { "name": "state", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[int]", "docstring": "Return the sequence axis for cache states that support block concat.", "summary": "Return the sequence axis for cache states that support block concat.", "implementation": "Method `BlockAwarePrefixCache._cache_state_seq_axis` calls `isinstance`, `len`, `hasattr`, `next`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 704, "end_line": 725, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L704-L725", "decorators": [], "calls": [ "isinstance", "len", "hasattr", "next", "iter" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "2", "1" ], "awaits": false, "yields": false }, { "name": "_slice_concat_cache_state", "qualname": "BlockAwarePrefixCache._slice_concat_cache_state", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache._slice_concat_cache_state", "kind": "method", "signature": "def _slice_concat_cache_state(self, state: Tuple[Any, ...] | List[Any], start_idx: int, end_idx: int) -> Tuple[Any, ...] | List[Any]", "parameters": [ { "name": "state", "kind": "positional or keyword", "annotation": "Tuple[Any, ...] | List[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "start_idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "end_idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Tuple[Any, ...] | List[Any]", "docstring": "Slice a sequence-backed cache state across the token axis.", "summary": "Slice a sequence-backed cache state across the token axis.", "implementation": "Method `BlockAwarePrefixCache._slice_concat_cache_state` calls `self._cache_state_seq_axis`, `ValueError`, `min`, `_slice_tensor`; can raise `ValueError`; returns `tuple(sliced) if isinstance(state, tuple) else sliced`.", "documented": true, "public": false, "addressable": true, "line": 727, "end_line": 751, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L727-L751", "decorators": [], "calls": [ "self._cache_state_seq_axis", "ValueError", "min", "_slice_tensor", "isinstance", "tuple" ], "state_reads": [ "self._cache_state_seq_axis" ], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "tuple(sliced) if isinstance(state, tuple) else sliced" ], "awaits": false, "yields": false }, { "name": "_slice_tensor", "qualname": "BlockAwarePrefixCache._slice_concat_cache_state._slice_tensor", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache._slice_concat_cache_state._slice_tensor", "kind": "nested function", "signature": "def _slice_tensor(tensor: Any) -> Any", "parameters": [ { "name": "tensor", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "", "summary": "Nested Function `BlockAwarePrefixCache._slice_concat_cache_state._slice_tensor` calls `slice`, `len`, `tuple`; returns `tensor[tuple(slices)]`.", "implementation": "Nested Function `BlockAwarePrefixCache._slice_concat_cache_state._slice_tensor` calls `slice`, `len`, `tuple`; returns `tensor[tuple(slices)]`.", "documented": false, "public": false, "addressable": false, "line": 745, "end_line": 748, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L745-L748", "decorators": [], "calls": [ "slice", "len", "tuple" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tensor[tuple(slices)]" ], "awaits": false, "yields": false }, { "name": "_concat_cache_states", "qualname": "BlockAwarePrefixCache._concat_cache_states", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache._concat_cache_states", "kind": "method", "signature": "def _concat_cache_states(self, states: List[Tuple[Any, ...] | List[Any]], seq_axis: int) -> Optional[Tuple[Any, ...] | List[Any]]", "parameters": [ { "name": "states", "kind": "positional or keyword", "annotation": "List[Tuple[Any, ...] | List[Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "seq_axis", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[Tuple[Any, ...] | List[Any]]", "docstring": "Concatenate state fragments for a sequence-backed cache layer.", "summary": "Concatenate state fragments for a sequence-backed cache layer.", "implementation": "Method `BlockAwarePrefixCache._concat_cache_states` calls `len`, `range`, `any`, `concatenated.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 753, "end_line": 768, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L753-L768", "decorators": [], "calls": [ "len", "range", "any", "concatenated.append", "mx.concatenate", "isinstance", "tuple" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "tuple(concatenated) if isinstance(states[0], tuple) else concatenated" ], "awaits": false, "yields": false }, { "name": "get_cache_for_generation", "qualname": "BlockAwarePrefixCache.get_cache_for_generation", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.get_cache_for_generation", "kind": "method", "signature": "def get_cache_for_generation(self, request_id: str) -> Tuple[Optional[List[Any]], bool]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Request identifier" } ], "return_annotation": "Tuple[Optional[List[Any]], bool]", "docstring": "Get cache data for generation, applying COW if needed.\n\nArgs:\n request_id: Request identifier\n\nReturns:\n Tuple of (cache_data, was_copied)", "summary": "Get cache data for generation, applying COW if needed.", "implementation": "Method `BlockAwarePrefixCache.get_cache_for_generation` calls `self._request_tables.get`, `self.paged_cache.get_blocks_for_generation`, `copy.deepcopy`, `time.time`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 770, "end_line": 799, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L770-L799", "decorators": [], "calls": [ "self._request_tables.get", "self.paged_cache.get_blocks_for_generation", "copy.deepcopy", "time.time" ], "state_reads": [ "self._request_tables.get", "self._request_tables", "self.paged_cache.get_blocks_for_generation", "self.paged_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "(None, False)", "(cache_data, was_copied)" ], "awaits": false, "yields": false }, { "name": "release_cache", "qualname": "BlockAwarePrefixCache.release_cache", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.release_cache", "kind": "method", "signature": "def release_cache(self, request_id: str) -> None", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Request identifier" } ], "return_annotation": "None", "docstring": "Release cache blocks for a completed request.\n\nArgs:\n request_id: Request identifier", "summary": "Release cache blocks for a completed request.", "implementation": "Method `BlockAwarePrefixCache.release_cache` calls `self._request_tables.pop`, `self.paged_cache.delete_block_table`, `logger.debug`.", "documented": true, "public": true, "addressable": true, "line": 801, "end_line": 811, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L801-L811", "decorators": [], "calls": [ "self._request_tables.pop", "self.paged_cache.delete_block_table", "logger.debug" ], "state_reads": [ "self._request_tables.pop", "self._request_tables", "self.paged_cache.delete_block_table", "self.paged_cache" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "fork_cache", "qualname": "BlockAwarePrefixCache.fork_cache", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.fork_cache", "kind": "method", "signature": "def fork_cache(self, source_request_id: str, new_request_id: str) -> Optional[BlockTable]", "parameters": [ { "name": "source_request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Source request ID" }, { "name": "new_request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "New request ID" } ], "return_annotation": "Optional[BlockTable]", "docstring": "Fork cache from one request to another (COW).\n\nArgs:\n source_request_id: Source request ID\n new_request_id: New request ID\n\nReturns:\n Forked BlockTable, or None if source not found", "summary": "Fork cache from one request to another (COW).", "implementation": "Method `BlockAwarePrefixCache.fork_cache` calls `self._request_tables.get`, `self.paged_cache.fork_block_table`, `BlockCacheEntry`, `time.time`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 813, "end_line": 847, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L813-L847", "decorators": [], "calls": [ "self._request_tables.get", "self.paged_cache.fork_block_table", "BlockCacheEntry", "time.time", "logger.debug" ], "state_reads": [ "self._request_tables.get", "self._request_tables", "self.paged_cache.fork_block_table", "self.paged_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "forked_table" ], "awaits": false, "yields": false }, { "name": "reconstruct_cache", "qualname": "BlockAwarePrefixCache.reconstruct_cache", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.reconstruct_cache", "kind": "method", "signature": "def reconstruct_cache(self, block_table: BlockTable) -> Optional[List[Any]]", "parameters": [ { "name": "block_table", "kind": "positional or keyword", "annotation": "BlockTable", "default": "", "required": true, "description": "BlockTable containing block IDs to reconstruct from" } ], "return_annotation": "Optional[List[Any]]", "docstring": "Reconstruct cache objects from stored block tensor data.\n\nSequence-backed caches are concatenated block-by-block. Recurrent\ncaches such as ArraysCache are restored from the latest sequence\nboundary snapshot that was actually stored.\n\nArgs:\n block_table: BlockTable containing block IDs to reconstruct from\n\nReturns:\n List of reconstructed KVCache objects (one per layer),\n or None if reconstruction fails", "summary": "Reconstruct cache objects from stored block tensor data.", "implementation": "Method `BlockAwarePrefixCache.reconstruct_cache` calls `logger.warning`, `self.paged_cache.allocated_blocks.get`, `logger.debug`, `all_block_data.append`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 849, "end_line": 967, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L849-L967", "decorators": [], "calls": [ "logger.warning", "self.paged_cache.allocated_blocks.get", "logger.debug", "all_block_data.append", "max", "len", "range", "self._concat_cache_states", "layer_meta.get", "hasattr", "_KVCache", "self._cache_state_seq_axis", "cache_cls.from_state", "KVCache", "reconstructed_caches.append", "traceback.format_exc" ], "state_reads": [ "self.paged_cache.allocated_blocks.get", "self.paged_cache.allocated_blocks", "self.paged_cache", "self._concat_cache_states", "self._cache_state_seq_axis" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "reconstructed_caches" ], "awaits": false, "yields": false }, { "name": "_find_best_prefix_match", "qualname": "BlockAwarePrefixCache._find_best_prefix_match", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache._find_best_prefix_match", "kind": "method", "signature": "def _find_best_prefix_match(self, tokens: List[int]) -> Optional[Tuple[List[int], List[int]]]", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[Tuple[List[int], List[int]]]", "docstring": "Find best matching prefix in the index.", "summary": "Find best matching prefix in the index.", "implementation": "Method `BlockAwarePrefixCache._find_best_prefix_match` calls `range`, `len`, `self.paged_cache.compute_block_hash`; returns `best_match`.", "documented": true, "public": false, "addressable": true, "line": 969, "end_line": 992, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L969-L992", "decorators": [], "calls": [ "range", "len", "self.paged_cache.compute_block_hash" ], "state_reads": [ "self.block_size", "self.paged_cache.compute_block_hash", "self.paged_cache", "self._prefix_index" ], "state_writes": [], "raises": [], "return_expressions": [ "best_match" ], "awaits": false, "yields": false }, { "name": "_update_prefix_index", "qualname": "BlockAwarePrefixCache._update_prefix_index", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache._update_prefix_index", "kind": "method", "signature": "def _update_prefix_index(self, tokens: List[int], block_ids: List[int]) -> None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "block_ids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Update prefix index with new token sequence.", "summary": "Update prefix index with new token sequence.", "implementation": "Method `BlockAwarePrefixCache._update_prefix_index` calls `range`, `len`, `min`, `self.paged_cache.compute_block_hash`.", "documented": true, "public": false, "addressable": true, "line": 994, "end_line": 1005, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L994-L1005", "decorators": [], "calls": [ "range", "len", "min", "self.paged_cache.compute_block_hash" ], "state_reads": [ "self.block_size", "self.paged_cache.compute_block_hash", "self.paged_cache", "self._prefix_index" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "BlockAwarePrefixCache.get_stats", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.get_stats", "kind": "method", "signature": "def get_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get cache statistics.", "summary": "Get cache statistics.", "implementation": "Method `BlockAwarePrefixCache.get_stats` calls `self.paged_cache.get_memory_usage`, `len`; returns `{'hits': self._hits, 'misses': self._misses, 'hit_rate': self._hits / (self._hits + self._misses) if self._hits + self.…`.", "documented": true, "public": true, "addressable": true, "line": 1007, "end_line": 1021, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L1007-L1021", "decorators": [], "calls": [ "self.paged_cache.get_memory_usage", "len" ], "state_reads": [ "self.paged_cache.get_memory_usage", "self.paged_cache", "self._hits", "self._misses", "self._tokens_saved", "self._request_tables" ], "state_writes": [], "raises": [], "return_expressions": [ "{'hits': self._hits, 'misses': self._misses, 'hit_rate': self._hits / (self._hits + self._misses) if self._hits + self.…" ], "awaits": false, "yields": false }, { "name": "reset_stats", "qualname": "BlockAwarePrefixCache.reset_stats", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.reset_stats", "kind": "method", "signature": "def reset_stats(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset statistics.", "summary": "Reset statistics.", "implementation": "Method `BlockAwarePrefixCache.reset_stats` updates `self._hits`, `self._misses`, `self._tokens_saved`; calls `self.paged_cache.reset_stats`.", "documented": true, "public": true, "addressable": true, "line": 1023, "end_line": 1028, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L1023-L1028", "decorators": [], "calls": [ "self.paged_cache.reset_stats" ], "state_reads": [ "self.paged_cache.reset_stats", "self.paged_cache" ], "state_writes": [ "self._hits", "self._misses", "self._tokens_saved" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "clear", "qualname": "BlockAwarePrefixCache.clear", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear all cached data.", "summary": "Clear all cached data.", "implementation": "Method `BlockAwarePrefixCache.clear` calls `self._request_tables.clear`, `self._prefix_index.clear`, `self.paged_cache.clear`, `self.reset_stats`.", "documented": true, "public": true, "addressable": true, "line": 1030, "end_line": 1035, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L1030-L1035", "decorators": [], "calls": [ "self._request_tables.clear", "self._prefix_index.clear", "self.paged_cache.clear", "self.reset_stats" ], "state_reads": [ "self._request_tables.clear", "self._request_tables", "self._prefix_index.clear", "self._prefix_index", "self.paged_cache.clear", "self.paged_cache", "self.reset_stats" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "BlockAwarePrefixCache.__len__", "full_name": "vllm_mlx.prefix_cache.BlockAwarePrefixCache.__len__", "kind": "method", "signature": "def __len__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Return number of active request entries.", "summary": "Return number of active request entries.", "implementation": "Method `BlockAwarePrefixCache.__len__` calls `len`; returns `len(self._request_tables)`.", "documented": true, "public": false, "addressable": true, "line": 1037, "end_line": 1039, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prefix_cache.py#L1037-L1039", "decorators": [], "calls": [ "len" ], "state_reads": [ "self._request_tables" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self._request_tables)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.prompt_warmup", "path": "vllm_mlx/prompt_warmup.py", "page_path": "reference/api/vllm_mlx/prompt_warmup.md", "docstring": "Prompt warm-up for vllm-mlx.\n\nAt server startup, pre-populates the prefix cache by running one short\ngeneration per warm-up prompt. The first user request that shares a prefix\nwith a warmed prompt sees cache-hit TTFT instead of cold prefill latency.\n\nFile format (JSON):\n [\n [{\"role\": \"system\", \"content\": \"You are ...\"}],\n [{\"role\": \"system\", \"content\": \"...\"}, {\"role\": \"user\", \"content\": \"hi\"}]\n ]\n\nEach entry is a list of chat messages — same shape as a ``/v1/chat/completions``\n``messages`` field. The warmer runs a ``max_tokens=1`` chat completion for each,\nwhich flows through the exact same path as a real request and writes the KV\nstate to the prefix cache.\n\nPaths resolve from the current working directory. A single-message system\nprompt is sufficient if that is the shared prefix.\n\nSizing note: prompts are warmed concurrently via ``asyncio.gather``, so N\nentries fire N concurrent prefills at startup. Each prefill allocates KV\ncache for its prompt length. For typical agent deployments 1–3 entries\n(one per active persona) cover the hot paths; a very large warm-prompts\nfile on a memory-tight model can exhaust headroom at boot.", "summary": "Prompt warm-up for vllm-mlx.", "line_count": 275, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L1-L275", "members": [ "logger", "load_warmup_file", "_ensure_user_terminator", "_build_strict_prefix_string", "warm_prefix_cache" ], "symbols": [ { "name": "load_warmup_file", "qualname": "load_warmup_file", "full_name": "vllm_mlx.prompt_warmup.load_warmup_file", "kind": "function", "signature": "def load_warmup_file(path: str) -> list[list[dict[str, Any]]]", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[list[dict[str, Any]]]", "docstring": "Load and validate a warm-up prompts JSON file.\n\nRaises:\n FileNotFoundError: If the file does not exist.\n ValueError: If the file shape is invalid.", "summary": "Load and validate a warm-up prompts JSON file.", "implementation": "Function `load_warmup_file` calls `Path(path).expanduser`, `Path`, `p.exists`, `FileNotFoundError`; can raise `FileNotFoundError`, `ValueError`; returns `data`.", "documented": true, "public": true, "addressable": true, "line": 41, "end_line": 76, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L41-L76", "decorators": [], "calls": [ "Path(path).expanduser", "Path", "p.exists", "FileNotFoundError", "json.loads", "p.read_text", "isinstance", "ValueError", "type", "enumerate" ], "state_reads": [], "state_writes": [], "raises": [ "FileNotFoundError", "ValueError" ], "return_expressions": [ "data" ], "awaits": false, "yields": false }, { "name": "_ensure_user_terminator", "qualname": "_ensure_user_terminator", "full_name": "vllm_mlx.prompt_warmup._ensure_user_terminator", "kind": "function", "signature": "def _ensure_user_terminator(messages: list[dict[str, Any]]) -> list[dict[str, Any]]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict[str, Any]]", "docstring": "Ensure the message list ends with a user message.\n\nSome chat templates (Qwen3.6, DeepSeek-VL, a handful of others) require\nat least one user message or raise ``TemplateError: No user query found``.\nWe prefer to cache just the system prefix, but when the template won't\nrender without a user, append a minimal placeholder. The common prefix\nup to the start of user content still matches real requests, so the\nsystem tokens still get cached.", "summary": "Ensure the message list ends with a user message.", "implementation": "Function `_ensure_user_terminator` calls `messages[-1].get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 79, "end_line": 91, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L79-L91", "decorators": [], "calls": [ "messages[-1].get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "messages", "[*messages, {'role': 'user', 'content': ' '}]" ], "awaits": false, "yields": false }, { "name": "_build_strict_prefix_string", "qualname": "_build_strict_prefix_string", "full_name": "vllm_mlx.prompt_warmup._build_strict_prefix_string", "kind": "function", "signature": "def _build_strict_prefix_string(tokenizer: Any, messages: list[dict[str, Any]], enable_thinking: bool=True) -> str | None", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "enable_thinking", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Optional positional or keyword input; defaults to `True`." } ], "return_annotation": "str | None", "docstring": "Build a STRING prefix that is a prefix of any real request's rendered\nchat template for the same system and empty chat history.\n\nStrategy: render the chat template twice with two DIFFERENT user contents\nand ``tokenize=False`` (matching what the server does). Truncate the\nfirst output at the position where the two strings diverge — that's\nwhere user content gets inserted.\n\nWe return a STRING (not tokens) because the engine's request path also\napplies the template with ``tokenize=False`` and then lets the tokenizer\nencode the result. Going through the same pipeline guarantees the warm\nentry's tokens are a strict prefix of a real request's tokens.\n\nThis enables warm-prompts on hybrid SSM+attention models where LCP\nmatching is disabled (SSM state can't be trimmed) — they rely purely\non strict PREFIX match.\n\nReturns None if rendering fails or the two probes don't diverge past a\nreasonable prefix length (unusual template).", "summary": "Build a STRING prefix that is a prefix of any real request's rendered chat template for the same system and empty chat history.", "implementation": "Function `_build_strict_prefix_string` calls `getattr`, `apply`, `_with_user`, `kwargs.pop`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 94, "end_line": 176, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L94-L176", "decorators": [], "calls": [ "getattr", "apply", "_with_user", "kwargs.pop", "isinstance", "range", "min", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "a[:boundary]" ], "awaits": false, "yields": false }, { "name": "_with_user", "qualname": "_build_strict_prefix_string._with_user", "full_name": "vllm_mlx.prompt_warmup._build_strict_prefix_string._with_user", "kind": "nested function", "signature": "def _with_user(user_content: str) -> list[dict[str, Any]]", "parameters": [ { "name": "user_content", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict[str, Any]]", "docstring": "", "summary": "Nested Function `_build_strict_prefix_string._with_user` calls `dict`, `msgs[-1].get`; returns `msgs`.", "implementation": "Nested Function `_build_strict_prefix_string._with_user` calls `dict`, `msgs[-1].get`; returns `msgs`.", "documented": false, "public": false, "addressable": false, "line": 121, "end_line": 127, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L121-L127", "decorators": [], "calls": [ "dict", "msgs[-1].get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "msgs" ], "awaits": false, "yields": false }, { "name": "warm_prefix_cache", "qualname": "warm_prefix_cache", "full_name": "vllm_mlx.prompt_warmup.warm_prefix_cache", "kind": "function", "signature": "async def warm_prefix_cache(engine: Any, prompts: list[list[dict[str, Any]]], *, max_tokens: int=1) -> dict[str, Any]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The vllm-mlx engine (exposes ``stream_chat`` and optionally ``tokenizer`` + ``stream_generate``)." }, { "name": "prompts", "kind": "positional or keyword", "annotation": "list[list[dict[str, Any]]]", "default": "", "required": true, "description": "List of message arrays." }, { "name": "max_tokens", "kind": "keyword-only", "annotation": "int", "default": "1", "required": false, "description": "Tokens to generate per warm-up. 1 is enough." } ], "return_annotation": "dict[str, Any]", "docstring": "Run each prompt through the engine to populate the prefix cache.\n\nPrefers the strict-prefix path when the engine exposes a tokenizer:\nmanually tokenize with ``add_generation_prompt=False`` and feed the\nraw token IDs to the engine's ``stream_generate`` (which accepts\n``prompt: str | list[int]``). Real requests — which always use\n``add_generation_prompt=True`` — will then find the warm entry as an\nexact strict prefix, independent of the engine's LCP matcher.\n\nThis is the difference between warm-prompts helping dense models only\nand helping hybrid SSM+attention models too.\n\nFalls back to ``engine.stream_chat`` with a placeholder user message\nappended if no tokenizer is exposed — strict-prefix match won't apply\nthere, so the feature is effectively LCP-only for that engine.\n\nRuns all prompts concurrently (``asyncio.gather``).\n\nArgs:\n engine: The vllm-mlx engine (exposes ``stream_chat`` and optionally\n ``tokenizer`` + ``stream_generate``).\n prompts: List of message arrays.\n max_tokens: Tokens to generate per warm-up. 1 is enough.\n\nReturns:\n Dict with ``count``, ``skipped``, ``elapsed_ms``,\n ``total_prompt_tokens``, and ``mode`` (``\"strict-prefix\"`` or\n ``\"chat-fallback\"``) describing which path was used.", "summary": "Run each prompt through the engine to populate the prefix cache.", "implementation": "Function `warm_prefix_cache` calls `getattr`, `hasattr`, `time.perf_counter`, `asyncio.gather`; awaits asynchronous work; returns `{'count': completed, 'skipped': skipped, 'elapsed_ms': elapsed_ms, 'total_prompt_tokens': total_prompt_tokens, 'mode': …`.", "documented": true, "public": true, "addressable": true, "line": 179, "end_line": 275, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L179-L275", "decorators": [], "calls": [ "getattr", "hasattr", "time.perf_counter", "asyncio.gather", "runner", "enumerate", "sum" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'count': completed, 'skipped': skipped, 'elapsed_ms': elapsed_ms, 'total_prompt_tokens': total_prompt_tokens, 'mode': …" ], "awaits": true, "yields": false }, { "name": "_one_strict", "qualname": "warm_prefix_cache._one_strict", "full_name": "vllm_mlx.prompt_warmup.warm_prefix_cache._one_strict", "kind": "nested function", "signature": "async def _one_strict(idx: int, messages: list[dict[str, Any]]) -> tuple[int, int, str | None]", "parameters": [ { "name": "idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[int, int, str | None]", "docstring": "", "summary": "Nested Function `warm_prefix_cache._one_strict` calls `_build_strict_prefix_string`, `_one_chat`, `engine.stream_generate`, `int`; awaits asynchronous work; has 3 explicit return paths.", "implementation": "Nested Function `warm_prefix_cache._one_strict` calls `_build_strict_prefix_string`, `_one_chat`, `engine.stream_generate`, `int`; awaits asynchronous work; has 3 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 217, "end_line": 235, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L217-L235", "decorators": [], "calls": [ "_build_strict_prefix_string", "_one_chat", "engine.stream_generate", "int", "type", "str", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "await _one_chat(idx, messages)", "(1, int(output.prompt_tokens or 0), None)", "(0, 0, 'no finished output')" ], "awaits": true, "yields": false }, { "name": "_one_chat", "qualname": "warm_prefix_cache._one_chat", "full_name": "vllm_mlx.prompt_warmup.warm_prefix_cache._one_chat", "kind": "nested function", "signature": "async def _one_chat(idx: int, messages: list[dict[str, Any]]) -> tuple[int, int, str | None]", "parameters": [ { "name": "idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[int, int, str | None]", "docstring": "", "summary": "Nested Function `warm_prefix_cache._one_chat` calls `_ensure_user_terminator`, `engine.stream_chat`, `int`, `type`; has 3 explicit return paths.", "implementation": "Nested Function `warm_prefix_cache._one_chat` calls `_ensure_user_terminator`, `engine.stream_chat`, `int`, `type`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 237, "end_line": 253, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/prompt_warmup.py#L237-L253", "decorators": [], "calls": [ "_ensure_user_terminator", "engine.stream_chat", "int", "type", "str", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(1, int(output.prompt_tokens or 0), None)", "(0, 0, 'no finished output')", "(0, 0, err)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning", "path": "vllm_mlx/reasoning/__init__.py", "page_path": "reference/api/vllm_mlx/reasoning/index.md", "docstring": "Reasoning parser module for vllm-mlx.\n\nThis module provides parsers for extracting reasoning/thinking content from\nmodel outputs. Supports models like Qwen3, DeepSeek-R1, etc. that use special\ntokens (e.g., ...) to separate reasoning from final responses.\n\nUsage:\n from vllm_mlx.reasoning import get_parser, list_parsers\n\n # Get a parser by name\n parser = get_parser(\"qwen3\")()\n\n # Extract reasoning from complete output\n reasoning, content = parser.extract_reasoning(model_output)\n\n # For streaming\n parser.reset_state()\n for delta in stream:\n msg = parser.extract_reasoning_streaming(prev, curr, delta)\n if msg:\n # msg.reasoning and/or msg.content will be populated\n ...", "summary": "Reasoning parser module for vllm-mlx.", "line_count": 110, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/__init__.py#L1-L110", "members": [ "_REASONING_PARSERS", "register_parser", "get_parser", "list_parsers", "_register_builtin_parsers", "__all__" ], "symbols": [ { "name": "register_parser", "qualname": "register_parser", "full_name": "vllm_mlx.reasoning.register_parser", "kind": "function", "signature": "def register_parser(name: str, parser_class: type[ReasoningParser]) -> None", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name to register the parser under (e.g., \"qwen3\")." }, { "name": "parser_class", "kind": "positional or keyword", "annotation": "type[ReasoningParser]", "default": "", "required": true, "description": "The parser class to register." } ], "return_annotation": "None", "docstring": "Register a reasoning parser.\n\nArgs:\n name: Name to register the parser under (e.g., \"qwen3\").\n parser_class: The parser class to register.", "summary": "Register a reasoning parser.", "implementation": "Function `register_parser` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 42, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/__init__.py#L34-L42", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_parser", "qualname": "get_parser", "full_name": "vllm_mlx.reasoning.get_parser", "kind": "function", "signature": "def get_parser(name: str) -> type[ReasoningParser]", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Name of the parser (e.g., \"qwen3\", \"deepseek_r1\")." } ], "return_annotation": "type[ReasoningParser]", "docstring": "Get a reasoning parser class by name.\n\nArgs:\n name: Name of the parser (e.g., \"qwen3\", \"deepseek_r1\").\n\nReturns:\n The parser class (not an instance).\n\nRaises:\n KeyError: If parser name is not found.", "summary": "Get a reasoning parser class by name.", "implementation": "Function `get_parser` calls `list`, `_REASONING_PARSERS.keys`, `KeyError`; can raise `KeyError`; returns `_REASONING_PARSERS[name]`.", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/__init__.py#L45-L63", "decorators": [], "calls": [ "list", "_REASONING_PARSERS.keys", "KeyError" ], "state_reads": [], "state_writes": [], "raises": [ "KeyError" ], "return_expressions": [ "_REASONING_PARSERS[name]" ], "awaits": false, "yields": false }, { "name": "list_parsers", "qualname": "list_parsers", "full_name": "vllm_mlx.reasoning.list_parsers", "kind": "function", "signature": "def list_parsers() -> list[str]", "parameters": [], "return_annotation": "list[str]", "docstring": "List available parser names.\n\nReturns:\n List of registered parser names.", "summary": "List available parser names.", "implementation": "Function `list_parsers` calls `list`, `_REASONING_PARSERS.keys`; returns `list(_REASONING_PARSERS.keys())`.", "documented": true, "public": true, "addressable": true, "line": 66, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/__init__.py#L66-L73", "decorators": [], "calls": [ "list", "_REASONING_PARSERS.keys" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "list(_REASONING_PARSERS.keys())" ], "awaits": false, "yields": false }, { "name": "_register_builtin_parsers", "qualname": "_register_builtin_parsers", "full_name": "vllm_mlx.reasoning._register_builtin_parsers", "kind": "function", "signature": "def _register_builtin_parsers()", "parameters": [], "return_annotation": "not annotated", "docstring": "Register built-in parsers.", "summary": "Register built-in parsers.", "implementation": "Function `_register_builtin_parsers` calls `register_parser`.", "documented": true, "public": false, "addressable": true, "line": 76, "end_line": 94, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/__init__.py#L76-L94", "decorators": [], "calls": [ "register_parser" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.base", "path": "vllm_mlx/reasoning/base.py", "page_path": "reference/api/vllm_mlx/reasoning/base.md", "docstring": "Base classes for reasoning content extraction.\n\nThis module provides the abstract base class for reasoning parsers that extract\nthinking/reasoning content from model outputs (e.g., ... tags).", "summary": "Base classes for reasoning content extraction.", "line_count": 126, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L1-L126", "members": [ "DeltaMessage", "ReasoningParser" ], "symbols": [ { "name": "DeltaMessage", "qualname": "DeltaMessage", "full_name": "vllm_mlx.reasoning.base.DeltaMessage", "kind": "class", "signature": "class DeltaMessage", "parameters": [ { "name": "role", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "content", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "reasoning", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "DeltaMessage", "docstring": "Delta message for streaming reasoning output.\n\nContains either reasoning content, regular content, or both when\ntransitioning from reasoning to content phase.\n\nNote: reasoning and content should typically not both be non-None\nexcept during the transition chunk.", "summary": "Delta message for streaming reasoning output.", "implementation": "Class `DeltaMessage` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 15, "end_line": 33, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L15-L33", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reasoning_content", "qualname": "DeltaMessage.reasoning_content", "full_name": "vllm_mlx.reasoning.base.DeltaMessage.reasoning_content", "kind": "method", "signature": "def reasoning_content(self) -> str | None", "parameters": [], "return_annotation": "str | None", "docstring": "Deprecated: use reasoning instead. Maintained for backward compatibility.", "summary": "Deprecated: use reasoning instead.", "implementation": "Method `DeltaMessage.reasoning_content` returns `self.reasoning`.", "documented": true, "public": true, "addressable": true, "line": 31, "end_line": 33, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L31-L33", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.reasoning" ], "state_writes": [], "raises": [], "return_expressions": [ "self.reasoning" ], "awaits": false, "yields": false }, { "name": "ReasoningParser", "qualname": "ReasoningParser", "full_name": "vllm_mlx.reasoning.base.ReasoningParser", "kind": "class", "signature": "class ReasoningParser(ABC)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional tokenizer for token-based parsing. For vllm-mlx, text-based parsing is sufficient, so this is optional." } ], "return_annotation": "ReasoningParser", "docstring": "Abstract base class for reasoning content extraction.\n\nReasoning parsers extract thinking/reasoning content from model outputs,\nseparating it from the final response content. This is useful for models\nlike DeepSeek-R1, Qwen3, etc. that use special tokens to denote reasoning.\n\nExample:\n Input: \"Let me solve this step by step...The answer is 42.\"\n Output: reasoning=\"Let me solve this step by step...\", content=\"The answer is 42.\"", "summary": "Abstract base class for reasoning content extraction.", "implementation": "Class `ReasoningParser` derives from `ABC` and declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 36, "end_line": 126, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L36-L126", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ReasoningParser.__init__", "full_name": "vllm_mlx.reasoning.base.ReasoningParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer: Any | None=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any | None", "default": "None", "required": false, "description": "Optional tokenizer for token-based parsing. For vllm-mlx, text-based parsing is sufficient, so this is optional." } ], "return_annotation": "not annotated", "docstring": "Initialize parser with optional tokenizer.\n\nArgs:\n tokenizer: Optional tokenizer for token-based parsing. For vllm-mlx,\n text-based parsing is sufficient, so this is optional.", "summary": "Initialize parser with optional tokenizer.", "implementation": "Method `ReasoningParser.__init__` updates `self.tokenizer`.", "documented": true, "public": false, "addressable": true, "line": 49, "end_line": 57, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L49-L57", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.tokenizer" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "ReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.base.ReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete text output from the model." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning content from complete model output.\n\nArgs:\n model_output: Complete text output from the model.\n\nReturns:\n Tuple of (reasoning_content, final_content).\n Either may be None if not present.", "summary": "Extract reasoning content from complete model output.", "implementation": "Method `ReasoningParser.extract_reasoning` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 60, "end_line": 74, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L60-L74", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_reasoning_streaming", "qualname": "ReasoningParser.extract_reasoning_streaming", "full_name": "vllm_mlx.reasoning.base.ReasoningParser.extract_reasoning_streaming", "kind": "method", "signature": "def extract_reasoning_streaming(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Accumulated text before this delta." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Accumulated text including this delta." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The new text in this streaming chunk." } ], "return_annotation": "DeltaMessage | None", "docstring": "Extract reasoning from streaming delta.\n\nUses the \"previous + delta = current\" model where:\n- previous_text: All text accumulated before this delta\n- current_text: All text including this delta (previous + delta)\n- delta_text: Just the new text in this chunk\n\nArgs:\n previous_text: Accumulated text before this delta.\n current_text: Accumulated text including this delta.\n delta_text: The new text in this streaming chunk.\n\nReturns:\n DeltaMessage with reasoning and/or content populated,\n or None if this delta should be skipped (e.g., special tokens).", "summary": "Extract reasoning from streaming delta.", "implementation": "Method `ReasoningParser.extract_reasoning_streaming` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 77, "end_line": 100, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L77-L100", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset_state", "qualname": "ReasoningParser.reset_state", "full_name": "vllm_mlx.reasoning.base.ReasoningParser.reset_state", "kind": "method", "signature": "def reset_state(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Reset any internal state for a new request.\n\nCalled before starting to process a new streaming request.\nOverride in subclasses if stateful parsing is needed.\nThis is intentionally a default no-op implementation.", "summary": "Reset any internal state for a new request.", "implementation": "Method `ReasoningParser.reset_state` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 102, "end_line": 110, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L102-L110", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "finalize_stream", "qualname": "ReasoningParser.finalize_stream", "full_name": "vllm_mlx.reasoning.base.ReasoningParser.finalize_stream", "kind": "method", "signature": "def finalize_stream(self) -> DeltaMessage | None", "parameters": [], "return_annotation": "DeltaMessage | None", "docstring": "Finalize streaming state at end of stream.\n\nCalled after the last delta is processed but before the stream\ncloses. Parsers that buffer partial markers internally should\nflush any remaining text here.\n\nDefault implementation is a no-op (returns None).\n\nReturns:\n DeltaMessage with any pending reasoning/content to emit,\n or None if nothing to flush.", "summary": "Finalize streaming state at end of stream.", "implementation": "Method `ReasoningParser.finalize_stream` returns `None`.", "documented": true, "public": true, "addressable": true, "line": 112, "end_line": 126, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/base.py#L112-L126", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.deepseek_r1_parser", "path": "vllm_mlx/reasoning/deepseek_r1_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/deepseek_r1_parser.md", "docstring": "Reasoning parser for DeepSeek-R1 models.\n\nDeepSeek-R1 uses ... tags for reasoning content.\nThe model may sometimes start outputting reasoning without the explicit\n tag, so this parser is more lenient than Qwen3.", "summary": "Reasoning parser for DeepSeek-R1 models.", "line_count": 114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/deepseek_r1_parser.py#L1-L114", "members": [ "DeepSeekR1ReasoningParser" ], "symbols": [ { "name": "DeepSeekR1ReasoningParser", "qualname": "DeepSeekR1ReasoningParser", "full_name": "vllm_mlx.reasoning.deepseek_r1_parser.DeepSeekR1ReasoningParser", "kind": "class", "signature": "class DeepSeekR1ReasoningParser(BaseThinkingReasoningParser)", "parameters": [], "return_annotation": "DeepSeekR1ReasoningParser", "docstring": " Reasoning parser for DeepSeek-R1 model.\n\n DeepSeek-R1 uses ... tokens to denote reasoning text.\n This parser is more lenient than Qwen3:\n - The tag may not be explicitly generated (model assumes it)\n - If only is found, everything before it is reasoning\n\n Example:\n Input: \"Step 1: analyze...\nStep 2: solve...The answer is 42.\"\n Output: reasoning=\"Step 1: analyze...\nStep 2: solve...\", content=\"The answer is 42.\"\n\n Input: \"reasoning contentfinal answer\" # No opening tag\n Output: reasoning=\"reasoning content\", content=\"final answer\"\n ", "summary": "Reasoning parser for DeepSeek-R1 model.", "implementation": "Class `DeepSeekR1ReasoningParser` derives from `BaseThinkingReasoningParser` and declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 14, "end_line": 114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/deepseek_r1_parser.py#L14-L114", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start_token", "qualname": "DeepSeekR1ReasoningParser.start_token", "full_name": "vllm_mlx.reasoning.deepseek_r1_parser.DeepSeekR1ReasoningParser.start_token", "kind": "method", "signature": "def start_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the marker that opens an explicit DeepSeek reasoning span.", "summary": "Return the marker that opens an explicit DeepSeek reasoning span.", "implementation": "Method `DeepSeekR1ReasoningParser.start_token` returns `''`.", "documented": true, "public": true, "addressable": true, "line": 32, "end_line": 35, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/deepseek_r1_parser.py#L32-L35", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false }, { "name": "end_token", "qualname": "DeepSeekR1ReasoningParser.end_token", "full_name": "vllm_mlx.reasoning.deepseek_r1_parser.DeepSeekR1ReasoningParser.end_token", "kind": "method", "signature": "def end_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the marker that closes a DeepSeek reasoning span.", "summary": "Return the marker that closes a DeepSeek reasoning span.", "implementation": "Method `DeepSeekR1ReasoningParser.end_token` returns `''`.", "documented": true, "public": true, "addressable": true, "line": 38, "end_line": 41, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/deepseek_r1_parser.py#L38-L41", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "DeepSeekR1ReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.deepseek_r1_parser.DeepSeekR1ReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete model output text." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning from DeepSeek-R1 output.\n\nMore lenient than Qwen3 - handles cases where start tag is implicit.\n\nArgs:\n model_output: Complete model output text.\n\nReturns:\n (reasoning, content) tuple.", "summary": "Extract reasoning from DeepSeek-R1 output.", "implementation": "Method `DeepSeekR1ReasoningParser.extract_reasoning` calls `self._extract_complete_reasoning`, `super().extract_reasoning`, `super`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 43, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/deepseek_r1_parser.py#L43-L67", "decorators": [], "calls": [ "self._extract_complete_reasoning", "super().extract_reasoning", "super" ], "state_reads": [ "self.end_token", "self.start_token", "self._extract_complete_reasoning" ], "state_writes": [], "raises": [], "return_expressions": [ "self._extract_complete_reasoning(model_output)", "(None, model_output)", "super().extract_reasoning(model_output)" ], "awaits": false, "yields": false }, { "name": "extract_reasoning_streaming", "qualname": "DeepSeekR1ReasoningParser.extract_reasoning_streaming", "full_name": "vllm_mlx.reasoning.deepseek_r1_parser.DeepSeekR1ReasoningParser.extract_reasoning_streaming", "kind": "method", "signature": "def extract_reasoning_streaming(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text accumulated before this delta." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text including this delta." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Just the new text." } ], "return_annotation": "DeltaMessage | None", "docstring": "Extract reasoning from streaming delta.\n\nHandles DeepSeek-R1's pattern where may be implicit.\n\nArgs:\n previous_text: Text accumulated before this delta.\n current_text: Text including this delta.\n delta_text: Just the new text.\n\nReturns:\n DeltaMessage with reasoning/content, or None to skip.", "summary": "Extract reasoning from streaming delta.", "implementation": "Method `DeepSeekR1ReasoningParser.extract_reasoning_streaming` calls `super().extract_reasoning_streaming`, `super`, `delta_text.find`, `len`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/deepseek_r1_parser.py#L69-L114", "decorators": [], "calls": [ "super().extract_reasoning_streaming", "super", "delta_text.find", "len", "DeltaMessage" ], "state_reads": [ "self.start_token", "self.end_token" ], "state_writes": [], "raises": [], "return_expressions": [ "DeltaMessage(reasoning=reasoning_part if reasoning_part else None, content=content_part if content_part else None)", "result" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.gemma4_parser", "path": "vllm_mlx/reasoning/gemma4_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/gemma4_parser.md", "docstring": "Reasoning parser for Gemma 4 models.\n\nGemma 4 uses a channel-based protocol for reasoning:\n\n <|channel>thought\n ...thinking content...\n \n ...response content...\n\nWhere:\n <|channel> = token 100 (channel switch marker)\n = token 101 (end-of-channel marker)\n\nThe channel names \"thought\" and \"response\" appear as text after the\nspecial tokens and should be stripped from the output.\n\nSome model variants may use <|channel>response instead of \nto transition from thinking to response mode. This parser handles both.\n\nWhen thinking is disabled or not triggered, output contains no tags.\n\nDegenerate cycling:\n On long prompts with tools, Gemma 4 may oscillate between thought and\n response channels many times, producing garbage reasoning before finally\n emitting valid content/tool_calls. The parser handles this by splitting\n at the LAST so all cycles go into reasoning_content and only\n the final response goes into content. Channel tokens are stripped from\n both sides.", "summary": "Reasoning parser for Gemma 4 models.", "line_count": 386, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L1-L386", "members": [ "_THOUGHT_PREFIX", "_RESPONSE_MARKER", "_THOUGHT_MARKER", "_strip_channel_name", "_strip_channel_tokens", "Gemma4ReasoningParser" ], "symbols": [ { "name": "_strip_channel_name", "qualname": "_strip_channel_name", "full_name": "vllm_mlx.reasoning.gemma4_parser._strip_channel_name", "kind": "function", "signature": "def _strip_channel_name(text: str, prefix: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prefix", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Strip channel name and leading whitespace/newline from text start.", "summary": "Strip channel name and leading whitespace/newline from text start.", "implementation": "Function `_strip_channel_name` calls `text.startswith`, `len`, `text.lstrip`; returns `text.lstrip('\\n')`.", "documented": true, "public": false, "addressable": true, "line": 46, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L46-L50", "decorators": [], "calls": [ "text.startswith", "len", "text.lstrip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text.lstrip('\\n')" ], "awaits": false, "yields": false }, { "name": "_strip_channel_tokens", "qualname": "_strip_channel_tokens", "full_name": "vllm_mlx.reasoning.gemma4_parser._strip_channel_tokens", "kind": "function", "signature": "def _strip_channel_tokens(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Remove all channel special tokens and bare channel names from text.\n\nHandles degenerate model output with multiple thought/response cycles\nby stripping all protocol tokens, leaving only the actual text content.", "summary": "Remove all channel special tokens and bare channel names from text.", "implementation": "Function `_strip_channel_tokens` calls `text.replace`, `text.split`, `line.strip`, `cleaned.append`; returns `text.strip()`.", "documented": true, "public": false, "addressable": true, "line": 53, "end_line": 82, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L53-L82", "decorators": [], "calls": [ "text.replace", "text.split", "line.strip", "cleaned.append", "'\\n'.join", "text.strip", "text.startswith", "len", "text[len(name)].isalpha" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text.strip()" ], "awaits": false, "yields": false }, { "name": "Gemma4ReasoningParser", "qualname": "Gemma4ReasoningParser", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser", "kind": "class", "signature": "class Gemma4ReasoningParser(BaseThinkingReasoningParser)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "Gemma4ReasoningParser", "docstring": "Reasoning parser for Gemma 4 models.\n\nHandles two transition formats:\n1. <|channel>thought...response (standard: token 100 + 101)\n2. <|channel>thought...<|channel>response (alternative: token 100 + 100)\n\nChannel names (\"thought\", \"response\") are stripped from output.\n\nExample:\n Input: \"<|channel>thought\\nLet me think...The answer is 42.\"\n Output: reasoning=\"Let me think...\", content=\"The answer is 42.\"\n\nWhen no tags are present, the entire output is treated as content.\n\nDegenerate cycling (long prompts + tools):\n Uses rpartition to split at the LAST , so all intermediate\n thought/response cycles go into reasoning and only the final response\n goes into content.\n\nStreaming buffering:\n Partial markers at a delta boundary (e.g. \"<|channel>\" without a\n following \"response\" yet) are buffered internally so they don't\n leak into reasoning/content. The buffer is either consumed when\n the marker completes in a later delta, or flushed as reasoning\n via finalize_stream() when the stream ends.", "summary": "Reasoning parser for Gemma 4 models.", "implementation": "Class `Gemma4ReasoningParser` derives from `BaseThinkingReasoningParser` and declares 10 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 85, "end_line": 386, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L85-L386", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start_token", "qualname": "Gemma4ReasoningParser.start_token", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser.start_token", "kind": "method", "signature": "def start_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return Gemma's marker for entering the thought channel.", "summary": "Return Gemma's marker for entering the thought channel.", "implementation": "Method `Gemma4ReasoningParser.start_token` returns `'<|channel>'`.", "documented": true, "public": true, "addressable": true, "line": 115, "end_line": 118, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L115-L118", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'<|channel>'" ], "awaits": false, "yields": false }, { "name": "end_token", "qualname": "Gemma4ReasoningParser.end_token", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser.end_token", "kind": "method", "signature": "def end_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return Gemma's marker for entering the response channel.", "summary": "Return Gemma's marker for entering the response channel.", "implementation": "Method `Gemma4ReasoningParser.end_token` returns `''`.", "documented": true, "public": true, "addressable": true, "line": 121, "end_line": 124, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L121-L124", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "Gemma4ReasoningParser.__init__", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `Gemma4ReasoningParser.__init__` updates `self._pending`, `self._content_seen`; calls `super().__init__`, `super`.", "implementation": "Method `Gemma4ReasoningParser.__init__` updates `self._pending`, `self._content_seen`; calls `super().__init__`, `super`.", "documented": false, "public": false, "addressable": true, "line": 126, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L126-L133", "decorators": [], "calls": [ "super().__init__", "super" ], "state_reads": [], "state_writes": [ "self._pending", "self._content_seen" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset_state", "qualname": "Gemma4ReasoningParser.reset_state", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser.reset_state", "kind": "method", "signature": "def reset_state(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Reset base parsing state and buffered Gemma channel markers.", "summary": "Reset base parsing state and buffered Gemma channel markers.", "implementation": "Method `Gemma4ReasoningParser.reset_state` updates `self._pending`, `self._content_seen`; calls `super().reset_state`, `super`.", "documented": true, "public": true, "addressable": true, "line": 135, "end_line": 140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L135-L140", "decorators": [], "calls": [ "super().reset_state", "super" ], "state_reads": [], "state_writes": [ "self._pending", "self._content_seen" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_trailing_partial_marker_len", "qualname": "Gemma4ReasoningParser._trailing_partial_marker_len", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser._trailing_partial_marker_len", "kind": "method", "signature": "def _trailing_partial_marker_len(self, text: str) -> int", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Return length of trailing substring of `text` that is a proper prefix\nof any transition marker (, <|channel>response, <|channel>).\n\nOnly counts PROPER prefixes — if the marker is already complete in\n`text`, no buffering is needed. Returns 0 if no partial match.\n\nWe must never buffer legitimate content. For <|channel>, only buffer\nwhen it appears AT THE END and is not followed by more text (i.e.,\n`response` or `thought` hasn't arrived yet).", "summary": "Return length of trailing substring of `text` that is a proper prefix of any transition marker (, <|channel>response, <|channel>).", "implementation": "Method `Gemma4ReasoningParser._trailing_partial_marker_len` calls `range`, `min`, `len`, `text.endswith`; returns `max_len`.", "documented": true, "public": false, "addressable": true, "line": 142, "end_line": 166, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L142-L166", "decorators": [], "calls": [ "range", "min", "len", "text.endswith" ], "state_reads": [ "self.end_token", "self.start_token" ], "state_writes": [], "raises": [], "return_expressions": [ "max_len" ], "awaits": false, "yields": false }, { "name": "finalize_stream", "qualname": "Gemma4ReasoningParser.finalize_stream", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser.finalize_stream", "kind": "method", "signature": "def finalize_stream(self) -> DeltaMessage | None", "parameters": [], "return_annotation": "DeltaMessage | None", "docstring": "Flush any buffered partial marker at the end of stream.\n\nIf the stream ends while we have a partial marker buffered (e.g.\nmodel emitted \"<|channel>\" as its last token and got truncated by\nmax_tokens), emit it as reasoning so the client doesn't lose the\ntext. Content phase flushes as content.", "summary": "Flush any buffered partial marker at the end of stream.", "implementation": "Method `Gemma4ReasoningParser.finalize_stream` updates `self._pending`; calls `DeltaMessage`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 168, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L168-L183", "decorators": [], "calls": [ "DeltaMessage" ], "state_reads": [ "self._pending", "self._phase" ], "state_writes": [ "self._pending" ], "raises": [], "return_expressions": [ "None", "DeltaMessage(content=pending)", "DeltaMessage(reasoning=pending)" ], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "Gemma4ReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning from complete output.\n\nUses rpartition (LAST ) to handle degenerate cycling:\nall intermediate thought/response cycles go into reasoning,\nonly the final response goes into content. Channel tokens\nare stripped from both sides.", "summary": "Extract reasoning from complete output.", "implementation": "Method `Gemma4ReasoningParser.extract_reasoning` calls `text.partition`, `after_start.rpartition`, `_strip_channel_tokens`, `text.count`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 185, "end_line": 233, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L185-L233", "decorators": [], "calls": [ "text.partition", "after_start.rpartition", "_strip_channel_tokens", "text.count", "after_start.rfind", "len", "text.rpartition" ], "state_reads": [ "self.start_token", "self.end_token" ], "state_writes": [], "raises": [], "return_expressions": [ "(reasoning or None, content or None)", "(reasoning or None, None)", "(None, model_output)" ], "awaits": false, "yields": false }, { "name": "extract_reasoning_streaming", "qualname": "Gemma4ReasoningParser.extract_reasoning_streaming", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser.extract_reasoning_streaming", "kind": "method", "signature": "def extract_reasoning_streaming(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "DeltaMessage | None", "docstring": "Extract reasoning from streaming delta.\n\nHandles:\n- No tags: treat as content (Gemma 4 doesn't inject tags in prompt)\n- <|channel>thought: enter reasoning mode, strip channel name\n- or <|channel>response: transition to content mode\n- Re-entry into thought from content (degenerate cycling): back to reasoning\n\nPartial markers at delta boundaries are buffered internally to\nprevent leaking them as reasoning/content.", "summary": "Extract reasoning from streaming delta.", "implementation": "Method `Gemma4ReasoningParser.extract_reasoning_streaming` updates `self._pending`; calls `self._trailing_partial_marker_len`, `len`, `self._extract_from_safe_text`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 235, "end_line": 272, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L235-L272", "decorators": [], "calls": [ "self._trailing_partial_marker_len", "len", "self._extract_from_safe_text" ], "state_reads": [ "self._trailing_partial_marker_len", "self._extract_from_safe_text" ], "state_writes": [ "self._pending" ], "raises": [], "return_expressions": [ "None", "self._extract_from_safe_text(safe_previous, safe_current, safe_delta)" ], "awaits": false, "yields": false }, { "name": "_strip_channel_tokens_from_delta", "qualname": "Gemma4ReasoningParser._strip_channel_tokens_from_delta", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser._strip_channel_tokens_from_delta", "kind": "method", "signature": "def _strip_channel_tokens_from_delta(msg: DeltaMessage | None) -> DeltaMessage | None", "parameters": [ { "name": "msg", "kind": "positional or keyword", "annotation": "DeltaMessage | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "DeltaMessage | None", "docstring": "Strip channel special tokens from content and reasoning in a delta.", "summary": "Strip channel special tokens from content and reasoning in a delta.", "implementation": "Method `Gemma4ReasoningParser._strip_channel_tokens_from_delta` calls `c.replace('', '').replace`, `c.replace`, `r.replace('', '').replace`, `r.replace`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 275, "end_line": 291, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L275-L291", "decorators": [ "staticmethod" ], "calls": [ "c.replace('', '').replace", "c.replace", "r.replace('', '').replace", "r.replace", "DeltaMessage" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "msg", "DeltaMessage(reasoning=r or None, content=c or None)" ], "awaits": false, "yields": false }, { "name": "_extract_from_safe_text", "qualname": "Gemma4ReasoningParser._extract_from_safe_text", "full_name": "vllm_mlx.reasoning.gemma4_parser.Gemma4ReasoningParser._extract_from_safe_text", "kind": "method", "signature": "def _extract_from_safe_text(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "DeltaMessage | None", "docstring": "Parse safe (non-buffered) text.\n\nUses count-based detection for channel tokens so that multiple\nthought/response cycles (degenerate model behaviour) are handled\ncorrectly — each NEW <|channel> re-enters reasoning, each NEW\n transitions to content.", "summary": "Parse safe (non-buffered) text.", "implementation": "Method `Gemma4ReasoningParser._extract_from_safe_text` updates `self._phase`, `self._content_seen`; calls `DeltaMessage`, `current_text.find`, `len`, `after_marker.lstrip`; has 8 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 293, "end_line": 386, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gemma4_parser.py#L293-L386", "decorators": [], "calls": [ "DeltaMessage", "current_text.find", "len", "after_marker.lstrip", "self._strip_channel_tokens_from_delta", "current_text.count", "previous_text.count", "delta_text.rfind", "_strip_channel_name", "after.lstrip", "delta_text.lstrip", "bool", "current_text.split", "after_ch.startswith", "after_ch[len(_THOUGHT_PREFIX):].lstrip", "previous_text.split", "prev_after.startswith", "prev_after[len(_THOUGHT_PREFIX):].lstrip" ], "state_reads": [ "self.start_token", "self.end_token", "self._strip_channel_tokens_from_delta", "self._phase", "self._content_seen" ], "state_writes": [ "self._phase", "self._content_seen" ], "raises": [], "return_expressions": [ "DeltaMessage(content=delta_text)", "self._strip_channel_tokens_from_delta(DeltaMessage(content=after_marker))", "None", "DeltaMessage(content=after)", "self._strip_channel_tokens_from_delta(DeltaMessage(content=stripped))", "self._strip_channel_tokens_from_delta(DeltaMessage(content=delta_text))", "DeltaMessage(reasoning=r) if r else None", "DeltaMessage(reasoning=delta_text) if delta_text else None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.glm4_parser", "path": "vllm_mlx/reasoning/glm4_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/glm4_parser.md", "docstring": "Reasoning parser for GLM-4 models (GLM-4.5-Air, GLM-4.6V, GLM-4.7, etc.).\n\nGLM-4 uses ... tags for reasoning content, same as Qwen3.\nHowever, unlike Qwen3, GLM-4 does NOT inject in the prompt —\nthe model decides autonomously whether to reason.\n\nThis means:\n- Output without tags = normal response (no reasoning)\n- Output with tags = reasoning + content\n\nThis is the opposite of Qwen3 where no tags = pure reasoning (because\n was injected in the prompt and the model hit max_tokens).\n\nGLM-4.6V also wraps responses in <|begin_of_box|>...<|end_of_box|> container\ntags which must be stripped before returning content.", "summary": "Reasoning parser for GLM-4 models (GLM-4.5-Air, GLM-4.6V, GLM-4.7, etc.).", "line_count": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/glm4_parser.py#L1-L113", "members": [ "_BOX_START", "_BOX_END", "Glm4ReasoningParser" ], "symbols": [ { "name": "Glm4ReasoningParser", "qualname": "Glm4ReasoningParser", "full_name": "vllm_mlx.reasoning.glm4_parser.Glm4ReasoningParser", "kind": "class", "signature": "class Glm4ReasoningParser(BaseThinkingReasoningParser)", "parameters": [], "return_annotation": "Glm4ReasoningParser", "docstring": "Reasoning parser for GLM-4 models.\n\nGLM-4 uses ... tokens to denote reasoning text.\nUnlike Qwen3, the template does NOT inject in the prompt,\nso output without tags is a normal response (not truncated reasoning).\n\nSupports three scenarios:\n1. Both tags in output: reasoningcontent\n2. Only closing tag (think in prompt): reasoningcontent\n3. No tags: pure content (NOT reasoning)\n\nExample (with thinking):\n Input: \"Let me analyze...The answer is 42.\"\n Output: reasoning=\"Let me analyze...\", content=\"The answer is 42.\"\n\nExample (no thinking):\n Input: \"The answer is 42.\"\n Output: reasoning=None, content=\"The answer is 42.\"", "summary": "Reasoning parser for GLM-4 models.", "implementation": "Class `Glm4ReasoningParser` derives from `BaseThinkingReasoningParser` and declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 27, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/glm4_parser.py#L27-L113", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start_token", "qualname": "Glm4ReasoningParser.start_token", "full_name": "vllm_mlx.reasoning.glm4_parser.Glm4ReasoningParser.start_token", "kind": "method", "signature": "def start_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the marker that opens a GLM reasoning span.", "summary": "Return the marker that opens a GLM reasoning span.", "implementation": "Method `Glm4ReasoningParser.start_token` returns `''`.", "documented": true, "public": true, "addressable": true, "line": 50, "end_line": 53, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/glm4_parser.py#L50-L53", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false }, { "name": "end_token", "qualname": "Glm4ReasoningParser.end_token", "full_name": "vllm_mlx.reasoning.glm4_parser.Glm4ReasoningParser.end_token", "kind": "method", "signature": "def end_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the marker that closes a GLM reasoning span.", "summary": "Return the marker that closes a GLM reasoning span.", "implementation": "Method `Glm4ReasoningParser.end_token` returns `''`.", "documented": true, "public": true, "addressable": true, "line": 56, "end_line": 59, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/glm4_parser.py#L56-L59", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "Glm4ReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.glm4_parser.Glm4ReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Strip GLM box markers and split complete reasoning from content.", "summary": "Strip GLM box markers and split complete reasoning from content.", "implementation": "Method `Glm4ReasoningParser.extract_reasoning` calls `model_output.replace(_BOX_START, '').replace`, `model_output.replace`, `super().extract_reasoning`, `super`; returns `super().extract_reasoning(cleaned)`.", "documented": true, "public": true, "addressable": true, "line": 61, "end_line": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/glm4_parser.py#L61-L68", "decorators": [], "calls": [ "model_output.replace(_BOX_START, '').replace", "model_output.replace", "super().extract_reasoning", "super" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "super().extract_reasoning(cleaned)" ], "awaits": false, "yields": false }, { "name": "extract_reasoning_streaming", "qualname": "Glm4ReasoningParser.extract_reasoning_streaming", "full_name": "vllm_mlx.reasoning.glm4_parser.Glm4ReasoningParser.extract_reasoning_streaming", "kind": "method", "signature": "def extract_reasoning_streaming(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "DeltaMessage | None", "docstring": "Extract reasoning from streaming delta.\n\nOverrides base class pre_think behavior: when no tags have been seen,\nemit delta as content (not reasoning). GLM-4 doesn't inject \nin the prompt, so early tokens without tags are normal content.\n\nOnce is seen, delegates to base class state machine.", "summary": "Extract reasoning from streaming delta.", "implementation": "Method `Glm4ReasoningParser.extract_reasoning_streaming` calls `delta_text.replace(_BOX_START, '').replace`, `delta_text.replace`, `super().extract_reasoning_streaming`, `super`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/glm4_parser.py#L70-L113", "decorators": [], "calls": [ "delta_text.replace(_BOX_START, '').replace", "delta_text.replace", "super().extract_reasoning_streaming", "super", "DeltaMessage" ], "state_reads": [ "self.start_token", "self.end_token", "self._phase" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "super().extract_reasoning_streaming(previous_text, current_text, delta_text)", "DeltaMessage(content=delta_text)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.gpt_oss_parser", "path": "vllm_mlx/reasoning/gpt_oss_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/gpt_oss_parser.md", "docstring": "Reasoning parser for GPT-OSS models using channel-based format.\n\nGPT-OSS models use a channel-based token format instead of ... tags:\n <|channel|>analysis<|message|>[reasoning]<|start|>assistant<|channel|>final<|message|>[content]<|return|>\n\nSome models also emit an extended format with a constrain token:\n <|channel|>final <|constrain|>JSON<|message|>[content]<|return|>\n\nThis parser extracts reasoning from the 'analysis' channel and content from\nthe 'final' channel, stripping all structural tokens from API responses.", "summary": "Reasoning parser for GPT-OSS models using channel-based format.", "line_count": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L1-L214", "members": [ "_STRUCTURAL_TOKENS", "_CHANNEL_RE", "_extract_channel", "GptOssReasoningParser" ], "symbols": [ { "name": "_extract_channel", "qualname": "_extract_channel", "full_name": "vllm_mlx.reasoning.gpt_oss_parser._extract_channel", "kind": "function", "signature": "def _extract_channel(text: str, channel_name: str) -> str | None", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Full model output text." }, { "name": "channel_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Channel name to extract (e.g., \"analysis\", \"final\")." } ], "return_annotation": "str | None", "docstring": "Extract content from a named channel.\n\nFinds <|channel|>{name}...<|message|> (with optional constrain token)\nand extracts text up to the next structural token or end of string.\n\nArgs:\n text: Full model output text.\n channel_name: Channel name to extract (e.g., \"analysis\", \"final\").\n\nReturns:\n Extracted channel content, or None if channel not found.", "summary": "Extract content from a named channel.", "implementation": "Function `_extract_channel` calls `_CHANNEL_RE.finditer`, `m.group`, `m.end`, `_STRUCTURAL_TOKENS.search`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 33, "end_line": 55, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L33-L55", "decorators": [], "calls": [ "_CHANNEL_RE.finditer", "m.group", "m.end", "_STRUCTURAL_TOKENS.search", "end_match.start", "content.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "content if content else None", "None" ], "awaits": false, "yields": false }, { "name": "GptOssReasoningParser", "qualname": "GptOssReasoningParser", "full_name": "vllm_mlx.reasoning.gpt_oss_parser.GptOssReasoningParser", "kind": "class", "signature": "class GptOssReasoningParser(ReasoningParser)", "parameters": [], "return_annotation": "GptOssReasoningParser", "docstring": "Reasoning parser for GPT-OSS models.\n\nGPT-OSS uses channel-based tokens:\n <|channel|>analysis<|message|>[reasoning]\n <|start|>assistant<|channel|>final<|message|>[content]<|return|>\n\nThe 'analysis' channel maps to reasoning, 'final' to content.\n\nAlso handles extended format with constrain token:\n <|channel|>final <|constrain|>JSON<|message|>[content]<|return|>", "summary": "Reasoning parser for GPT-OSS models.", "implementation": "Class `GptOssReasoningParser` derives from `ReasoningParser` and declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 58, "end_line": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L58-L214", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "GptOssReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.gpt_oss_parser.GptOssReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete text output from the model." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning and content from complete model output.\n\nArgs:\n model_output: Complete text output from the model.\n\nReturns:\n (reasoning, content) tuple. Either may be None.", "summary": "Extract reasoning and content from complete model output.", "implementation": "Method `GptOssReasoningParser.extract_reasoning` calls `_extract_channel`, `content.replace('<|return|>', '').strip`, `content.replace`, `_STRUCTURAL_TOKENS.sub('', content).strip`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 72, "end_line": 106, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L72-L106", "decorators": [], "calls": [ "_extract_channel", "content.replace('<|return|>', '').strip", "content.replace", "_STRUCTURAL_TOKENS.sub('', content).strip", "_STRUCTURAL_TOKENS.sub", "_STRUCTURAL_TOKENS.sub('', reasoning).strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(None, model_output if model_output else None)", "(None, model_output)", "(reasoning, content)" ], "awaits": false, "yields": false }, { "name": "extract_reasoning_streaming", "qualname": "GptOssReasoningParser.extract_reasoning_streaming", "full_name": "vllm_mlx.reasoning.gpt_oss_parser.GptOssReasoningParser.extract_reasoning_streaming", "kind": "method", "signature": "def extract_reasoning_streaming(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Accumulated text before this delta." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Accumulated text including this delta." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Just the new text in this streaming chunk." } ], "return_annotation": "DeltaMessage | None", "docstring": "Extract reasoning from streaming delta.\n\nUses stateless phase detection from current_text on each call.\n\nArgs:\n previous_text: Accumulated text before this delta.\n current_text: Accumulated text including this delta.\n delta_text: Just the new text in this streaming chunk.\n\nReturns:\n DeltaMessage with reasoning and/or content, or None to skip.", "summary": "Extract reasoning from streaming delta.", "implementation": "Method `GptOssReasoningParser.extract_reasoning_streaming` calls `self._detect_phase`, `self._extract_content_after_marker_in_delta`, `self._strip_return`, `DeltaMessage`; has 5 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 108, "end_line": 161, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L108-L161", "decorators": [], "calls": [ "self._detect_phase", "self._extract_content_after_marker_in_delta", "self._strip_return", "DeltaMessage", "_STRUCTURAL_TOKENS.search", "_STRUCTURAL_TOKENS.sub" ], "state_reads": [ "self._detect_phase", "self._extract_content_after_marker_in_delta", "self._strip_return" ], "state_writes": [], "raises": [], "return_expressions": [ "DeltaMessage(reasoning=after_marker)", "DeltaMessage(content=after_marker)", "None", "DeltaMessage(reasoning=cleaned)", "DeltaMessage(content=cleaned)" ], "awaits": false, "yields": false }, { "name": "_detect_phase", "qualname": "GptOssReasoningParser._detect_phase", "full_name": "vllm_mlx.reasoning.gpt_oss_parser.GptOssReasoningParser._detect_phase", "kind": "method", "signature": "def _detect_phase(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Detect current streaming phase from accumulated text.\n\nReturns:\n \"final\" — final channel marker complete\n \"analysis\" — analysis marker complete, no structural token after\n \"transition\" — analysis present but structural token follows\n \"init\" — no channel marker yet", "summary": "Detect current streaming phase from accumulated text.", "implementation": "Method `GptOssReasoningParser._detect_phase` calls `list`, `_CHANNEL_RE.finditer`, `last.group`, `last.end`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 164, "end_line": 187, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L164-L187", "decorators": [ "staticmethod" ], "calls": [ "list", "_CHANNEL_RE.finditer", "last.group", "last.end", "_STRUCTURAL_TOKENS.search" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'init'", "'final'", "'transition'", "'analysis'" ], "awaits": false, "yields": false }, { "name": "_extract_content_after_marker_in_delta", "qualname": "GptOssReasoningParser._extract_content_after_marker_in_delta", "full_name": "vllm_mlx.reasoning.gpt_oss_parser.GptOssReasoningParser._extract_content_after_marker_in_delta", "kind": "method", "signature": "def _extract_content_after_marker_in_delta(current_text: str, phase: str) -> str | None", "parameters": [ { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Full accumulated text." }, { "name": "phase", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Current phase (\"analysis\" or \"final\")." } ], "return_annotation": "str | None", "docstring": "When phase changes, extract only the content after the phase marker\nthat falls within the current accumulated text's tail.\n\nArgs:\n current_text: Full accumulated text.\n phase: Current phase (\"analysis\" or \"final\").\n\nReturns:\n Content after the marker, or None.", "summary": "When phase changes, extract only the content after the phase marker that falls within the current accumulated text's tail.", "implementation": "Method `GptOssReasoningParser._extract_content_after_marker_in_delta` calls `list`, `_CHANNEL_RE.finditer`, `reversed`, `m.group`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 190, "end_line": 209, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L190-L209", "decorators": [ "staticmethod" ], "calls": [ "list", "_CHANNEL_RE.finditer", "reversed", "m.group", "m.end" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "current_text[m.end():]", "None" ], "awaits": false, "yields": false }, { "name": "_strip_return", "qualname": "GptOssReasoningParser._strip_return", "full_name": "vllm_mlx.reasoning.gpt_oss_parser.GptOssReasoningParser._strip_return", "kind": "method", "signature": "def _strip_return(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Strip <|return|> from text.", "summary": "Strip <|return|> from text.", "implementation": "Method `GptOssReasoningParser._strip_return` calls `text.replace`; returns `text.replace('<|return|>', '')`.", "documented": true, "public": false, "addressable": true, "line": 212, "end_line": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/gpt_oss_parser.py#L212-L214", "decorators": [ "staticmethod" ], "calls": [ "text.replace" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text.replace('<|return|>', '')" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.harmony_parser", "path": "vllm_mlx/reasoning/harmony_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/harmony_parser.md", "docstring": "Reasoning parser for GPT-OSS models using Harmony format.\n\nHarmony uses channels for reasoning vs final content:\n\n <|channel|>analysis\n <|message|>Let me think about this...\n <|end|>\n <|channel|>final\n <|message|>The answer is 42.\n <|return|>\n\nThe analysis channel contains reasoning, and the final channel\ncontains the user-facing response.", "summary": "Reasoning parser for GPT-OSS models using Harmony format.", "line_count": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/harmony_parser.py#L1-L157", "members": [ "_ANALYSIS_PATTERN", "_FINAL_PATTERN", "HarmonyReasoningParser" ], "symbols": [ { "name": "HarmonyReasoningParser", "qualname": "HarmonyReasoningParser", "full_name": "vllm_mlx.reasoning.harmony_parser.HarmonyReasoningParser", "kind": "class", "signature": "class HarmonyReasoningParser(ReasoningParser)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "HarmonyReasoningParser", "docstring": "Reasoning parser for GPT-OSS models using Harmony format.\n\nExtracts reasoning from the 'analysis' channel and content from\nthe 'final' channel. Commentary channels (tool calls) are ignored\nsince they are handled by the tool parser.\n\nExample:\n Input: \"<|channel|>analysis<|message|>Thinking...<|end|>\n <|channel|>final<|message|>Result.<|return|>\"\n Output: reasoning=\"Thinking...\", content=\"Result.\"", "summary": "Reasoning parser for GPT-OSS models using Harmony format.", "implementation": "Class `HarmonyReasoningParser` derives from `ReasoningParser` and declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 35, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/harmony_parser.py#L35-L157", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "HarmonyReasoningParser.__init__", "full_name": "vllm_mlx.reasoning.harmony_parser.HarmonyReasoningParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `HarmonyReasoningParser.__init__` updates `self._current_channel`, `self._in_message`; calls `super().__init__`, `super`.", "implementation": "Method `HarmonyReasoningParser.__init__` updates `self._current_channel`, `self._in_message`; calls `super().__init__`, `super`.", "documented": false, "public": false, "addressable": true, "line": 49, "end_line": 52, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/harmony_parser.py#L49-L52", "decorators": [], "calls": [ "super().__init__", "super" ], "state_reads": [], "state_writes": [ "self._current_channel", "self._in_message" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "HarmonyReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.harmony_parser.HarmonyReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete model output text." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning from complete Harmony output.\n\nCollects all analysis channel blocks as reasoning and the\nfinal channel block as content.\n\nArgs:\n model_output: Complete model output text.\n\nReturns:\n (reasoning, content) tuple. Either may be None.", "summary": "Extract reasoning from complete Harmony output.", "implementation": "Method `HarmonyReasoningParser.extract_reasoning` calls `_ANALYSIS_PATTERN.findall`, `'\\n'.join`, `block.strip`, `_FINAL_PATTERN.search`; returns `(reasoning, content)`.", "documented": true, "public": true, "addressable": true, "line": 54, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/harmony_parser.py#L54-L78", "decorators": [], "calls": [ "_ANALYSIS_PATTERN.findall", "'\\n'.join", "block.strip", "_FINAL_PATTERN.search", "final_match.group(1).strip", "final_match.group" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(reasoning, content)" ], "awaits": false, "yields": false }, { "name": "extract_reasoning_streaming", "qualname": "HarmonyReasoningParser.extract_reasoning_streaming", "full_name": "vllm_mlx.reasoning.harmony_parser.HarmonyReasoningParser.extract_reasoning_streaming", "kind": "method", "signature": "def extract_reasoning_streaming(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Accumulated text before this delta." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Accumulated text including this delta." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The new text in this streaming chunk." } ], "return_annotation": "DeltaMessage | None", "docstring": "Extract reasoning from streaming Harmony output.\n\nTracks the current channel and emits reasoning deltas for\nanalysis channel content and content deltas for final channel.\n\nArgs:\n previous_text: Accumulated text before this delta.\n current_text: Accumulated text including this delta.\n delta_text: The new text in this streaming chunk.\n\nReturns:\n DeltaMessage with reasoning and/or content, or None.", "summary": "Extract reasoning from streaming Harmony output.", "implementation": "Method `HarmonyReasoningParser.extract_reasoning_streaming` updates `self._current_channel`, `self._in_message`; calls `current_text.rfind`, `len`, `after.startswith`, `any`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 80, "end_line": 152, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/harmony_parser.py#L80-L152", "decorators": [], "calls": [ "current_text.rfind", "len", "after.startswith", "any", "delta_text.strip().startswith", "delta_text.strip", "delta_text.strip().endswith", "DeltaMessage" ], "state_reads": [ "self._current_channel", "self._in_message" ], "state_writes": [ "self._current_channel", "self._in_message" ], "raises": [], "return_expressions": [ "None", "DeltaMessage(reasoning=delta_text)", "DeltaMessage(content=delta_text)" ], "awaits": false, "yields": false }, { "name": "reset_state", "qualname": "HarmonyReasoningParser.reset_state", "full_name": "vllm_mlx.reasoning.harmony_parser.HarmonyReasoningParser.reset_state", "kind": "method", "signature": "def reset_state(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Reset streaming state for a new request.", "summary": "Reset streaming state for a new request.", "implementation": "Method `HarmonyReasoningParser.reset_state` updates `self._current_channel`, `self._in_message`.", "documented": true, "public": true, "addressable": true, "line": 154, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/harmony_parser.py#L154-L157", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._current_channel", "self._in_message" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.mistral_parser", "path": "vllm_mlx/reasoning/mistral_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/mistral_parser.md", "docstring": "Reasoning parser for Mistral / Ministral reasoning models.\n\nModels such as Magistral and Ministral-3-*-Reasoning wrap their reasoning in\n[THINK]...[/THINK] delimiters (registered as special tokens in the tokenizer)\nand support a strict switch via 'enable_thinking=False' in chat template kwargs.\n\nThis mirrors the Qwen3 parser, which uses ..., but with the\nMistral bracket-token delimiters.\n\nSupports implicit reasoning mode where [THINK] is injected in the prompt and\nonly [/THINK] appears in the output.", "summary": "Reasoning parser for Mistral / Ministral reasoning models.", "line_count": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/mistral_parser.py#L1-L72", "members": [ "MistralReasoningParser" ], "symbols": [ { "name": "MistralReasoningParser", "qualname": "MistralReasoningParser", "full_name": "vllm_mlx.reasoning.mistral_parser.MistralReasoningParser", "kind": "class", "signature": "class MistralReasoningParser(BaseThinkingReasoningParser)", "parameters": [], "return_annotation": "MistralReasoningParser", "docstring": "Reasoning parser for Mistral/Ministral reasoning models.\n\nUses [THINK]...[/THINK] tokens to denote reasoning text.\n\nSupports three scenarios:\n1. Both tags in output: [THINK]reasoning[/THINK]content\n2. Only closing tag (think in prompt): reasoning[/THINK]content\n3. No tags: pure content\n\nExample (normal):\n Input: \"[THINK]Let me analyze this...[/THINK]The answer is 42.\"\n Output: reasoning=\"Let me analyze this...\", content=\"The answer is 42.\"\n\nExample (think in prompt):\n Input: \"Let me analyze this...[/THINK]The answer is 42.\"\n Output: reasoning=\"Let me analyze this...\", content=\"The answer is 42.\"", "summary": "Reasoning parser for Mistral/Ministral reasoning models.", "implementation": "Class `MistralReasoningParser` derives from `BaseThinkingReasoningParser` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 19, "end_line": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/mistral_parser.py#L19-L72", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start_token", "qualname": "MistralReasoningParser.start_token", "full_name": "vllm_mlx.reasoning.mistral_parser.MistralReasoningParser.start_token", "kind": "method", "signature": "def start_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the Mistral reasoning opening marker.", "summary": "Return the Mistral reasoning opening marker.", "implementation": "Method `MistralReasoningParser.start_token` returns `'[THINK]'`.", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 43, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/mistral_parser.py#L40-L43", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'[THINK]'" ], "awaits": false, "yields": false }, { "name": "end_token", "qualname": "MistralReasoningParser.end_token", "full_name": "vllm_mlx.reasoning.mistral_parser.MistralReasoningParser.end_token", "kind": "method", "signature": "def end_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the Mistral reasoning closing marker.", "summary": "Return the Mistral reasoning closing marker.", "implementation": "Method `MistralReasoningParser.end_token` returns `'[/THINK]'`.", "documented": true, "public": true, "addressable": true, "line": 46, "end_line": 49, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/mistral_parser.py#L46-L49", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'[/THINK]'" ], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "MistralReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.mistral_parser.MistralReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete model output text." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning from Mistral/Ministral output.\n\nHandles both explicit [THINK]...[/THINK] tags and implicit mode\nwhere [THINK] was in the prompt (only [/THINK] in output).\n\nArgs:\n model_output: Complete model output text.\n\nReturns:\n (reasoning, content) tuple.", "summary": "Extract reasoning from Mistral/Ministral output.", "implementation": "Method `MistralReasoningParser.extract_reasoning` calls `super().extract_reasoning`, `super`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 51, "end_line": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/mistral_parser.py#L51-L72", "decorators": [], "calls": [ "super().extract_reasoning", "super" ], "state_reads": [ "self.end_token" ], "state_writes": [], "raises": [], "return_expressions": [ "(None, model_output)", "super().extract_reasoning(model_output)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.poolside_v1_parser", "path": "vllm_mlx/reasoning/poolside_v1_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/poolside_v1_parser.md", "docstring": "Reasoning parser for Poolside Laguna models.", "summary": "Reasoning parser for Poolside Laguna models.", "line_count": 13, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/poolside_v1_parser.py#L1-L13", "members": [ "PoolsideV1ReasoningParser" ], "symbols": [ { "name": "PoolsideV1ReasoningParser", "qualname": "PoolsideV1ReasoningParser", "full_name": "vllm_mlx.reasoning.poolside_v1_parser.PoolsideV1ReasoningParser", "kind": "class", "signature": "class PoolsideV1ReasoningParser(Qwen3ReasoningParser)", "parameters": [], "return_annotation": "PoolsideV1ReasoningParser", "docstring": "Parse Laguna's template-injected ```` reasoning boundary.\n\nvllm-mlx reasoning parsers receive only the generated assistant text, not\nprompt-history token IDs. Historical ```` markers therefore cannot\naffect this output-scoped parser as they can in token-aware vLLM serving.", "summary": "Parse Laguna's template-injected ```` reasoning boundary.", "implementation": "Class `PoolsideV1ReasoningParser` derives from `Qwen3ReasoningParser` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 7, "end_line": 13, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/poolside_v1_parser.py#L7-L13", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.qwen3_parser", "path": "vllm_mlx/reasoning/qwen3_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/qwen3_parser.md", "docstring": "Reasoning parser for Qwen3 models.\n\nQwen3 uses ... tags for reasoning content and supports\na strict switch via 'enable_thinking=False' in chat template kwargs.\n\nSupports implicit reasoning mode where is injected in the prompt\nby AI agents (e.g., OpenCode) and only appears in the output.", "summary": "Reasoning parser for Qwen3 models.", "line_count": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/qwen3_parser.py#L1-L68", "members": [ "Qwen3ReasoningParser" ], "symbols": [ { "name": "Qwen3ReasoningParser", "qualname": "Qwen3ReasoningParser", "full_name": "vllm_mlx.reasoning.qwen3_parser.Qwen3ReasoningParser", "kind": "class", "signature": "class Qwen3ReasoningParser(BaseThinkingReasoningParser)", "parameters": [], "return_annotation": "Qwen3ReasoningParser", "docstring": "Reasoning parser for Qwen3 models.\n\nQwen3 uses ... tokens to denote reasoning text.\n\nSupports three scenarios:\n1. Both tags in output: reasoningcontent\n2. Only closing tag (think in prompt): reasoningcontent\n3. No tags: pure content\n\nExample (normal):\n Input: \"Let me analyze this...The answer is 42.\"\n Output: reasoning=\"Let me analyze this...\", content=\"The answer is 42.\"\n\nExample (think in prompt):\n Input: \"Let me analyze this...The answer is 42.\"\n Output: reasoning=\"Let me analyze this...\", content=\"The answer is 42.\"", "summary": "Reasoning parser for Qwen3 models.", "implementation": "Class `Qwen3ReasoningParser` derives from `BaseThinkingReasoningParser` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 15, "end_line": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/qwen3_parser.py#L15-L68", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start_token", "qualname": "Qwen3ReasoningParser.start_token", "full_name": "vllm_mlx.reasoning.qwen3_parser.Qwen3ReasoningParser.start_token", "kind": "method", "signature": "def start_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the Qwen3 reasoning opening marker.", "summary": "Return the Qwen3 reasoning opening marker.", "implementation": "Method `Qwen3ReasoningParser.start_token` returns `''`.", "documented": true, "public": true, "addressable": true, "line": 36, "end_line": 39, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/qwen3_parser.py#L36-L39", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false }, { "name": "end_token", "qualname": "Qwen3ReasoningParser.end_token", "full_name": "vllm_mlx.reasoning.qwen3_parser.Qwen3ReasoningParser.end_token", "kind": "method", "signature": "def end_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the Qwen3 reasoning closing marker.", "summary": "Return the Qwen3 reasoning closing marker.", "implementation": "Method `Qwen3ReasoningParser.end_token` returns `''`.", "documented": true, "public": true, "addressable": true, "line": 42, "end_line": 45, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/qwen3_parser.py#L42-L45", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "Qwen3ReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.qwen3_parser.Qwen3ReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete model output text." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning from Qwen3 output.\n\nHandles both explicit ... tags and implicit mode\nwhere was in the prompt (only in output).\n\nArgs:\n model_output: Complete model output text.\n\nReturns:\n (reasoning, content) tuple.", "summary": "Extract reasoning from Qwen3 output.", "implementation": "Method `Qwen3ReasoningParser.extract_reasoning` calls `super().extract_reasoning`, `super`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 47, "end_line": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/qwen3_parser.py#L47-L68", "decorators": [], "calls": [ "super().extract_reasoning", "super" ], "state_reads": [ "self.end_token" ], "state_writes": [], "raises": [], "return_expressions": [ "(None, model_output)", "super().extract_reasoning(model_output)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.reasoning.think_parser", "path": "vllm_mlx/reasoning/think_parser.py", "page_path": "reference/api/vllm_mlx/reasoning/think_parser.md", "docstring": "Base parser for models using ... tags for reasoning.\n\nThis module provides BaseThinkingReasoningParser, a concrete implementation\nfor extracting reasoning content from models that use thinking tags.\n\nSupports three scenarios:\n1. Both tags in output: reasoningcontent\n2. Only closing tag (think injected in prompt): reasoningcontent\n3. No tags: pure content\n\nPerformance: The streaming parser uses a simple state machine to track the\ncurrent phase (pre-think / thinking / content). Tag completion is detected\nagainst the accumulated text for correctness when `` / `` are\nsplit across delta boundaries, but phase tracking still avoids the old\nwhole-output rescanning behavior.", "summary": "Base parser for models using ... tags for reasoning.", "line_count": 462, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L1-L462", "members": [ "logger", "BaseThinkingReasoningParser" ], "symbols": [ { "name": "BaseThinkingReasoningParser", "qualname": "BaseThinkingReasoningParser", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser", "kind": "class", "signature": "class BaseThinkingReasoningParser(ReasoningParser)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "BaseThinkingReasoningParser", "docstring": "Base parser for models using ... style tags.\n\nThis parser handles the common pattern where reasoning content is wrapped\nin special tags. Subclasses define the specific start and end tokens.\n\nSupports \"implicit reasoning mode\" where is injected in the prompt\nand only appears in the model output. This is common with AI agents\nlike OpenCode that force models to reason by injecting thinking tags.\n\nThe streaming parser uses a state machine with three phases:\n\n pre_think -> thinking -> content\n\nTransitions are tracked by parser state. Accumulated text is consulted only\nto detect when a start/end tag has completed across delta boundaries.", "summary": "Base parser for models using ... style tags.", "implementation": "Class `BaseThinkingReasoningParser` derives from `ReasoningParser` and declares 12 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 29, "end_line": 462, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L29-L462", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "start_token", "qualname": "BaseThinkingReasoningParser.start_token", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser.start_token", "kind": "method", "signature": "def start_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "The token/tag that starts reasoning content (e.g., '').", "summary": "The token/tag that starts reasoning content (e.g., '').", "implementation": "Method `BaseThinkingReasoningParser.start_token` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 50, "end_line": 51, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L50-L51", "decorators": [ "property", "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "end_token", "qualname": "BaseThinkingReasoningParser.end_token", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser.end_token", "kind": "method", "signature": "def end_token(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "The token/tag that ends reasoning content (e.g., '').", "summary": "The token/tag that ends reasoning content (e.g., '').", "implementation": "Method `BaseThinkingReasoningParser.end_token` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 55, "end_line": 56, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L55-L56", "decorators": [ "property", "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "BaseThinkingReasoningParser.__init__", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `BaseThinkingReasoningParser.__init__` updates `self._phase`, `self._content_started`, `self._content_buffer`, `self._in_tool_call`; calls `super().__init__`, `super`.", "implementation": "Method `BaseThinkingReasoningParser.__init__` updates `self._phase`, `self._content_started`, `self._content_buffer`, `self._in_tool_call`; calls `super().__init__`, `super`.", "documented": false, "public": false, "addressable": true, "line": 63, "end_line": 71, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L63-L71", "decorators": [], "calls": [ "super().__init__", "super" ], "state_reads": [], "state_writes": [ "self._phase", "self._content_started", "self._content_buffer", "self._in_tool_call", "self._tool_call_buffer" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset_state", "qualname": "BaseThinkingReasoningParser.reset_state", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser.reset_state", "kind": "method", "signature": "def reset_state(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Reset state machine for a new streaming request.", "summary": "Reset state machine for a new streaming request.", "implementation": "Method `BaseThinkingReasoningParser.reset_state` updates `self._phase`, `self._content_started`, `self._content_buffer`, `self._in_tool_call`.", "documented": true, "public": true, "addressable": true, "line": 73, "end_line": 79, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L73-L79", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._phase", "self._content_started", "self._content_buffer", "self._in_tool_call", "self._tool_call_buffer" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_reasoning", "qualname": "BaseThinkingReasoningParser.extract_reasoning", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser.extract_reasoning", "kind": "method", "signature": "def extract_reasoning(self, model_output: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete model output text." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Extract reasoning from complete output.\n\nHandles three cases:\n1. Both tags present: reasoningcontent\n2. Only closing tag: reasoningcontent (think in prompt)\n3. No tags: pure content\n\nArgs:\n model_output: Complete model output text.\n\nReturns:\n (reasoning, content) tuple. Either may be None.", "summary": "Extract reasoning from complete output.", "implementation": "Method `BaseThinkingReasoningParser.extract_reasoning` calls `self._extract_complete_reasoning`, `self._promote_tool_calls`, `text.partition`, `reasoning.strip`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 81, "end_line": 110, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L81-L110", "decorators": [], "calls": [ "self._extract_complete_reasoning", "self._promote_tool_calls", "text.partition", "reasoning.strip" ], "state_reads": [ "self.end_token", "self._extract_complete_reasoning", "self._promote_tool_calls", "self.start_token" ], "state_writes": [], "raises": [], "return_expressions": [ "self._promote_tool_calls(reasoning, content)", "self._promote_tool_calls(reasoning, None)", "(None, model_output)" ], "awaits": false, "yields": false }, { "name": "extract_reasoning_streaming", "qualname": "BaseThinkingReasoningParser.extract_reasoning_streaming", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser.extract_reasoning_streaming", "kind": "method", "signature": "def extract_reasoning_streaming(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text accumulated before this delta." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text including this delta." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Just the new text in this chunk." } ], "return_annotation": "DeltaMessage | None", "docstring": "Extract reasoning from a streaming delta using state-machine tracking.\n\nInstead of rescanning the full accumulated text on every token, this\nmethod tracks the current phase (pre_think / thinking / content) and\nonly consults accumulated text to detect completed start/end tags that\nwere split across delta boundaries.\n\nHandles three scenarios:\n1. Explicit ... in model output\n2. Implicit mode ( in prompt, only in output)\n3. No tags at all (pure content after first token with no reasoning)\n\nArgs:\n previous_text: Text accumulated before this delta.\n current_text: Text including this delta.\n delta_text: Just the new text in this chunk.\n\nReturns:\n DeltaMessage with reasoning and/or content, or None to skip.", "summary": "Extract reasoning from a streaming delta using state-machine tracking.", "implementation": "Method `BaseThinkingReasoningParser.extract_reasoning_streaming` updates `self._phase`, `self._in_tool_call`, `self._tool_call_buffer`; calls `delta_text.find`, `len`, `after.find`, `self._transition_to_content`; has 8 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 112, "end_line": 224, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L112-L224", "decorators": [], "calls": [ "delta_text.find", "len", "after.find", "self._transition_to_content", "DeltaMessage", "self._thinking_tool_call", "self._content_delta" ], "state_reads": [ "self.start_token", "self.end_token", "self._phase", "self._transition_to_content", "self._TOOL_CALL_START", "self._in_tool_call", "self._thinking_tool_call", "self._content_delta" ], "state_writes": [ "self._phase", "self._in_tool_call", "self._tool_call_buffer" ], "raises": [], "return_expressions": [ "None", "self._transition_to_content(reasoning, content)", "DeltaMessage(reasoning=before) if before else None", "DeltaMessage(reasoning=after) if after else None", "DeltaMessage(reasoning=delta_text)", "self._thinking_tool_call(previous_text, current_text, delta_text)", "DeltaMessage(reasoning=reasoning) if reasoning else None", "self._content_delta(delta_text)" ], "awaits": false, "yields": false }, { "name": "_extract_complete_reasoning", "qualname": "BaseThinkingReasoningParser._extract_complete_reasoning", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser._extract_complete_reasoning", "kind": "method", "signature": "def _extract_complete_reasoning(self, text: str) -> tuple[str | None, str | None]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Split complete output into leading reasoning spans and final content.", "summary": "Split complete output into leading reasoning spans and final content.", "implementation": "Method `BaseThinkingReasoningParser._extract_complete_reasoning` calls `remainder.lstrip`, `stripped.startswith`, `len`, `after_start.partition`; returns `(reasoning, content)`.", "documented": true, "public": false, "addressable": true, "line": 226, "end_line": 260, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L226-L260", "decorators": [], "calls": [ "remainder.lstrip", "stripped.startswith", "len", "after_start.partition", "reasoning_parts.append", "reasoning.strip", "stripped.find", "'\\n'.join(reasoning_parts).strip", "'\\n'.join", "remainder.strip" ], "state_reads": [ "self.start_token", "self.end_token" ], "state_writes": [], "raises": [], "return_expressions": [ "(reasoning, content)" ], "awaits": false, "yields": false }, { "name": "_transition_to_content", "qualname": "BaseThinkingReasoningParser._transition_to_content", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser._transition_to_content", "kind": "method", "signature": "def _transition_to_content(self, reasoning: str | None, content: str | None) -> DeltaMessage | None", "parameters": [ { "name": "reasoning", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "content", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "DeltaMessage | None", "docstring": "Return a delta while suppressing leading post-transition think blocks.", "summary": "Return a delta while suppressing leading post-transition think blocks.", "implementation": "Method `BaseThinkingReasoningParser._transition_to_content` calls `self._promote_tool_calls`, `self._content_delta`, `DeltaMessage`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 262, "end_line": 276, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L262-L276", "decorators": [], "calls": [ "self._promote_tool_calls", "self._content_delta", "DeltaMessage" ], "state_reads": [ "self._promote_tool_calls", "self._content_delta" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "DeltaMessage(reasoning=reasoning_text or None, content=final_content or None)" ], "awaits": false, "yields": false }, { "name": "_content_delta", "qualname": "BaseThinkingReasoningParser._content_delta", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser._content_delta", "kind": "method", "signature": "def _content_delta(self, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "DeltaMessage | None", "docstring": "Emit content after consuming repeated leading think blocks.", "summary": "Emit content after consuming repeated leading think blocks.", "implementation": "Method `BaseThinkingReasoningParser._content_delta` updates `self._content_buffer`, `self._content_started`; calls `DeltaMessage`, `self._content_buffer.lstrip`, `buffer.startswith`, `buffer[len(self.end_token):].lstrip`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 278, "end_line": 325, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L278-L325", "decorators": [], "calls": [ "DeltaMessage", "self._content_buffer.lstrip", "buffer.startswith", "buffer[len(self.end_token):].lstrip", "len", "after_start.find", "reasoning_parts.append", "after_start[end_idx + len(self.end_token):].lstrip", "self.start_token.startswith", "self.end_token.startswith", "''.join" ], "state_reads": [ "self._content_buffer", "self._content_started", "self._content_buffer.lstrip", "self.end_token", "self.start_token", "self.start_token.startswith", "self.end_token.startswith" ], "state_writes": [ "self._content_buffer", "self._content_started" ], "raises": [], "return_expressions": [ "None", "DeltaMessage(content=delta_text) if delta_text else None", "DeltaMessage(reasoning=''.join(reasoning_parts) or None, content=buffer)", "DeltaMessage(reasoning=''.join(reasoning_parts))" ], "awaits": false, "yields": false }, { "name": "_thinking_tool_call", "qualname": "BaseThinkingReasoningParser._thinking_tool_call", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser._thinking_tool_call", "kind": "method", "signature": "def _thinking_tool_call(self, previous_text: str, current_text: str, delta_text: str) -> DeltaMessage | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "DeltaMessage | None", "docstring": "Handle streaming while inside a during thinking phase.", "summary": "Handle streaming while inside a during thinking phase.", "implementation": "Method `BaseThinkingReasoningParser._thinking_tool_call` updates `self._tool_call_buffer`, `self._in_tool_call`, `self._phase`; calls `self._tool_call_buffer.find`, `len`, `logger.warning`, `remainder.find`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 327, "end_line": 396, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L327-L396", "decorators": [], "calls": [ "self._tool_call_buffer.find", "len", "logger.warning", "remainder.find", "remainder[:eidx].strip", "self._content_delta", "DeltaMessage", "remainder[:tc_idx].strip", "remainder.strip" ], "state_reads": [ "self._TOOL_CALL_END", "self.end_token", "self._tool_call_buffer.find", "self._tool_call_buffer", "self._content_delta", "self._TOOL_CALL_START" ], "state_writes": [ "self._tool_call_buffer", "self._in_tool_call", "self._phase" ], "raises": [], "return_expressions": [ "DeltaMessage(content=final_content or None, reasoning=r_text or None)", "DeltaMessage(content=promoted, reasoning=reasoning)", "DeltaMessage(content=final_content or None)", "None" ], "awaits": false, "yields": false }, { "name": "finalize_stream", "qualname": "BaseThinkingReasoningParser.finalize_stream", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser.finalize_stream", "kind": "method", "signature": "def finalize_stream(self) -> DeltaMessage | None", "parameters": [], "return_annotation": "DeltaMessage | None", "docstring": "Flush any buffered tool call text at end of stream.", "summary": "Flush any buffered tool call text at end of stream.", "implementation": "Method `BaseThinkingReasoningParser.finalize_stream` updates `self._tool_call_buffer`, `self._in_tool_call`; calls `logger.warning`, `DeltaMessage`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 398, "end_line": 406, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L398-L406", "decorators": [], "calls": [ "logger.warning", "DeltaMessage" ], "state_reads": [ "self._in_tool_call", "self._tool_call_buffer" ], "state_writes": [ "self._tool_call_buffer", "self._in_tool_call" ], "raises": [], "return_expressions": [ "DeltaMessage(content=promoted)", "None" ], "awaits": false, "yields": false }, { "name": "_promote_tool_calls", "qualname": "BaseThinkingReasoningParser._promote_tool_calls", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser._promote_tool_calls", "kind": "method", "signature": "def _promote_tool_calls(cls, reasoning: str | None, content: str | None) -> tuple[str | None, str | None]", "parameters": [ { "name": "reasoning", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "content", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "", "summary": "Method `BaseThinkingReasoningParser._promote_tool_calls` calls `cls._TOOL_CALL_CLOSED_RE.sub`, `cls._TOOL_CALL_UNCLOSED_RE.search`, `unclosed_match.group`, `unclosed_match.start`; has 2 explicit return paths.", "implementation": "Method `BaseThinkingReasoningParser._promote_tool_calls` calls `cls._TOOL_CALL_CLOSED_RE.sub`, `cls._TOOL_CALL_UNCLOSED_RE.search`, `unclosed_match.group`, `unclosed_match.start`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 409, "end_line": 462, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L409-L462", "decorators": [ "classmethod" ], "calls": [ "cls._TOOL_CALL_CLOSED_RE.sub", "cls._TOOL_CALL_UNCLOSED_RE.search", "unclosed_match.group", "unclosed_match.start", "cleaned.strip", "len", "'\\n'.join", "result_content.strip", "logger.warning" ], "state_reads": [ "cls._TOOL_CALL_CLOSED_RE.sub", "cls._TOOL_CALL_CLOSED_RE", "cls._TOOL_CALL_UNCLOSED_RE.search", "cls._TOOL_CALL_UNCLOSED_RE" ], "state_writes": [], "raises": [], "return_expressions": [ "(reasoning, content)", "(cleaned, result_content)" ], "awaits": false, "yields": false }, { "name": "_collect_closed", "qualname": "BaseThinkingReasoningParser._promote_tool_calls._collect_closed", "full_name": "vllm_mlx.reasoning.think_parser.BaseThinkingReasoningParser._promote_tool_calls._collect_closed", "kind": "nested function", "signature": "def _collect_closed(match)", "parameters": [ { "name": "match", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `BaseThinkingReasoningParser._promote_tool_calls._collect_closed` calls `closed.append`, `match.group`; returns `''`.", "implementation": "Nested Function `BaseThinkingReasoningParser._promote_tool_calls._collect_closed` calls `closed.append`, `match.group`; returns `''`.", "documented": false, "public": false, "addressable": false, "line": 419, "end_line": 421, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/reasoning/think_parser.py#L419-L421", "decorators": [], "calls": [ "closed.append", "match.group" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.request", "path": "vllm_mlx/request.py", "page_path": "reference/api/vllm_mlx/request.md", "docstring": "Request management for vllm-mlx continuous batching.\n\nThis module provides Request and RequestStatus classes adapted from vLLM's\nrequest management system, simplified for MLX backend.", "summary": "Request management for vllm-mlx continuous batching.", "line_count": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L1-L227", "members": [ "RequestStatus", "SamplingParams", "Request", "RequestOutput" ], "symbols": [ { "name": "RequestStatus", "qualname": "RequestStatus", "full_name": "vllm_mlx.request.RequestStatus", "kind": "class", "signature": "class RequestStatus(enum.IntEnum)", "parameters": [], "return_annotation": "RequestStatus", "docstring": "Status of a request in the scheduling system.", "summary": "Status of a request in the scheduling system.", "implementation": "Class `RequestStatus` derives from `enum.IntEnum` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 18, "end_line": 48, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L18-L48", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_finished", "qualname": "RequestStatus.is_finished", "full_name": "vllm_mlx.request.RequestStatus.is_finished", "kind": "method", "signature": "def is_finished(status: 'RequestStatus') -> bool", "parameters": [ { "name": "status", "kind": "positional or keyword", "annotation": "'RequestStatus'", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if the status indicates a finished request.", "summary": "Check if the status indicates a finished request.", "implementation": "Method `RequestStatus.is_finished` returns `status > RequestStatus.PREEMPTED`.", "documented": true, "public": true, "addressable": true, "line": 35, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L35-L37", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "status > RequestStatus.PREEMPTED" ], "awaits": false, "yields": false }, { "name": "get_finish_reason", "qualname": "RequestStatus.get_finish_reason", "full_name": "vllm_mlx.request.RequestStatus.get_finish_reason", "kind": "method", "signature": "def get_finish_reason(status: 'RequestStatus') -> Optional[str]", "parameters": [ { "name": "status", "kind": "positional or keyword", "annotation": "'RequestStatus'", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[str]", "docstring": "Get the finish reason string for a finished status.", "summary": "Get the finish reason string for a finished status.", "implementation": "Method `RequestStatus.get_finish_reason` has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 48, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L40-L48", "decorators": [ "staticmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'stop'", "'length'", "'abort'", "None" ], "awaits": false, "yields": false }, { "name": "SamplingParams", "qualname": "SamplingParams", "full_name": "vllm_mlx.request.SamplingParams", "kind": "class", "signature": "class SamplingParams", "parameters": [ { "name": "max_tokens", "kind": "field", "annotation": "int", "default": "256", "required": false, "description": "Optional constructor field; defaults to `256`." }, { "name": "temperature", "kind": "field", "annotation": "float", "default": "0.7", "required": false, "description": "Optional constructor field; defaults to `0.7`." }, { "name": "top_p", "kind": "field", "annotation": "float", "default": "0.9", "required": false, "description": "Optional constructor field; defaults to `0.9`." }, { "name": "top_k", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "min_p", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "presence_penalty", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "repetition_penalty", "kind": "field", "annotation": "float", "default": "1.0", "required": false, "description": "Optional constructor field; defaults to `1.0`." }, { "name": "stop", "kind": "field", "annotation": "Optional[List[str]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "stop_token_ids", "kind": "field", "annotation": "Optional[List[int]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "logits_processors", "kind": "field", "annotation": "Optional[List[Callable]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "SamplingParams", "docstring": "Sampling parameters for text generation.", "summary": "Sampling parameters for text generation.", "implementation": "Class `SamplingParams` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 52, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L52-L73", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__post_init__", "qualname": "SamplingParams.__post_init__", "full_name": "vllm_mlx.request.SamplingParams.__post_init__", "kind": "method", "signature": "def __post_init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `SamplingParams.__post_init__` updates `self.stop`, `self.stop_token_ids`.", "implementation": "Method `SamplingParams.__post_init__` updates `self.stop`, `self.stop_token_ids`.", "documented": false, "public": false, "addressable": true, "line": 69, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L69-L73", "decorators": [], "calls": [], "state_reads": [ "self.stop", "self.stop_token_ids" ], "state_writes": [ "self.stop", "self.stop_token_ids" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "Request", "qualname": "Request", "full_name": "vllm_mlx.request.Request", "kind": "class", "signature": "class Request", "parameters": [ { "name": "request_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "prompt", "kind": "field", "annotation": "Union[str, List[int]]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "sampling_params", "kind": "field", "annotation": "SamplingParams", "default": "", "required": true, "description": "Required constructor field." }, { "name": "arrival_time", "kind": "field", "annotation": "float", "default": "field(default_factory=time.time)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=time.time)`." }, { "name": "priority", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "prompt_token_ids", "kind": "field", "annotation": "Optional[List[int]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "num_prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "status", "kind": "field", "annotation": "RequestStatus", "default": "RequestStatus.WAITING", "required": false, "description": "Optional constructor field; defaults to `RequestStatus.WAITING`." }, { "name": "num_computed_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "output_token_ids", "kind": "field", "annotation": "List[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "output_text", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "batch_uid", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prompt_cache", "kind": "field", "annotation": "Optional[List[Any]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "cached_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "remaining_tokens", "kind": "field", "annotation": "Optional[List[int]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prefix_boundary", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "block_table", "kind": "field", "annotation": "Optional['BlockTable']", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "shared_prefix_blocks", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "images", "kind": "field", "annotation": "Optional[List[Any]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "videos", "kind": "field", "annotation": "Optional[List[Any]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "pixel_values", "kind": "field", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "image_grid_thw", "kind": "field", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "attention_mask", "kind": "field", "annotation": "Optional[Any]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "multimodal_kwargs", "kind": "field", "annotation": "Optional[Dict[str, Any]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "is_multimodal", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "finish_reason", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "first_token_time", "kind": "field", "annotation": "Optional[float]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "cache_hit_type", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "Request", "docstring": "Represents a single inference request in the scheduling system.\n\nAdapted from vLLM's Request class with simplifications for MLX backend.\n\nAttributes:\n request_id: Unique identifier for this request\n prompt: The input prompt (string or token ids)\n prompt_token_ids: Tokenized prompt\n sampling_params: Parameters for generation\n arrival_time: When the request was received\n status: Current status of the request\n num_prompt_tokens: Number of tokens in the prompt\n num_computed_tokens: Number of tokens processed so far\n output_token_ids: Generated token ids\n output_text: Generated text (decoded)", "summary": "Represents a single inference request in the scheduling system.", "implementation": "Class `Request` declares 10 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 77, "end_line": 192, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L77-L192", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "num_output_tokens", "qualname": "Request.num_output_tokens", "full_name": "vllm_mlx.request.Request.num_output_tokens", "kind": "method", "signature": "def num_output_tokens(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Number of output tokens generated so far.", "summary": "Number of output tokens generated so far.", "implementation": "Method `Request.num_output_tokens` calls `len`; returns `len(self.output_token_ids)`.", "documented": true, "public": true, "addressable": true, "line": 146, "end_line": 148, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L146-L148", "decorators": [ "property" ], "calls": [ "len" ], "state_reads": [ "self.output_token_ids" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.output_token_ids)" ], "awaits": false, "yields": false }, { "name": "num_tokens", "qualname": "Request.num_tokens", "full_name": "vllm_mlx.request.Request.num_tokens", "kind": "method", "signature": "def num_tokens(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Total number of tokens (prompt + output).", "summary": "Total number of tokens (prompt + output).", "implementation": "Method `Request.num_tokens` returns `self.num_prompt_tokens + self.num_output_tokens`.", "documented": true, "public": true, "addressable": true, "line": 151, "end_line": 153, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L151-L153", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.num_prompt_tokens", "self.num_output_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "self.num_prompt_tokens + self.num_output_tokens" ], "awaits": false, "yields": false }, { "name": "max_tokens", "qualname": "Request.max_tokens", "full_name": "vllm_mlx.request.Request.max_tokens", "kind": "method", "signature": "def max_tokens(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Maximum output tokens for this request.", "summary": "Maximum output tokens for this request.", "implementation": "Method `Request.max_tokens` returns `self.sampling_params.max_tokens`.", "documented": true, "public": true, "addressable": true, "line": 156, "end_line": 158, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L156-L158", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.sampling_params.max_tokens", "self.sampling_params" ], "state_writes": [], "raises": [], "return_expressions": [ "self.sampling_params.max_tokens" ], "awaits": false, "yields": false }, { "name": "is_finished", "qualname": "Request.is_finished", "full_name": "vllm_mlx.request.Request.is_finished", "kind": "method", "signature": "def is_finished(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if request has finished.", "summary": "Check if request has finished.", "implementation": "Method `Request.is_finished` calls `RequestStatus.is_finished`; returns `RequestStatus.is_finished(self.status)`.", "documented": true, "public": true, "addressable": true, "line": 160, "end_line": 162, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L160-L162", "decorators": [], "calls": [ "RequestStatus.is_finished" ], "state_reads": [ "self.status" ], "state_writes": [], "raises": [], "return_expressions": [ "RequestStatus.is_finished(self.status)" ], "awaits": false, "yields": false }, { "name": "get_finish_reason", "qualname": "Request.get_finish_reason", "full_name": "vllm_mlx.request.Request.get_finish_reason", "kind": "method", "signature": "def get_finish_reason(self) -> Optional[str]", "parameters": [], "return_annotation": "Optional[str]", "docstring": "Get the finish reason if finished.", "summary": "Get the finish reason if finished.", "implementation": "Method `Request.get_finish_reason` calls `RequestStatus.get_finish_reason`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 164, "end_line": 168, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L164-L168", "decorators": [], "calls": [ "RequestStatus.get_finish_reason" ], "state_reads": [ "self.finish_reason", "self.status" ], "state_writes": [], "raises": [], "return_expressions": [ "self.finish_reason", "RequestStatus.get_finish_reason(self.status)" ], "awaits": false, "yields": false }, { "name": "append_output_token", "qualname": "Request.append_output_token", "full_name": "vllm_mlx.request.Request.append_output_token", "kind": "method", "signature": "def append_output_token(self, token_id: int) -> None", "parameters": [ { "name": "token_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Append a generated token to the output.", "summary": "Append a generated token to the output.", "implementation": "Method `Request.append_output_token` updates `self.num_computed_tokens`; calls `self.output_token_ids.append`.", "documented": true, "public": true, "addressable": true, "line": 170, "end_line": 173, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L170-L173", "decorators": [], "calls": [ "self.output_token_ids.append" ], "state_reads": [ "self.output_token_ids.append", "self.output_token_ids" ], "state_writes": [ "self.num_computed_tokens" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "set_finished", "qualname": "Request.set_finished", "full_name": "vllm_mlx.request.Request.set_finished", "kind": "method", "signature": "def set_finished(self, status: RequestStatus, reason: Optional[str]=None) -> None", "parameters": [ { "name": "status", "kind": "positional or keyword", "annotation": "RequestStatus", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "reason", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Mark the request as finished.", "summary": "Mark the request as finished.", "implementation": "Method `Request.set_finished` updates `self.status`, `self.finish_reason`; calls `RequestStatus.get_finish_reason`.", "documented": true, "public": true, "addressable": true, "line": 175, "end_line": 178, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L175-L178", "decorators": [], "calls": [ "RequestStatus.get_finish_reason" ], "state_reads": [], "state_writes": [ "self.status", "self.finish_reason" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__lt__", "qualname": "Request.__lt__", "full_name": "vllm_mlx.request.Request.__lt__", "kind": "method", "signature": "def __lt__(self, other: 'Request') -> bool", "parameters": [ { "name": "other", "kind": "positional or keyword", "annotation": "'Request'", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Compare requests for priority queue ordering.", "summary": "Compare requests for priority queue ordering.", "implementation": "Method `Request.__lt__` has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 180, "end_line": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L180-L184", "decorators": [], "calls": [], "state_reads": [ "self.priority", "self.arrival_time" ], "state_writes": [], "raises": [], "return_expressions": [ "self.priority < other.priority", "self.arrival_time < other.arrival_time" ], "awaits": false, "yields": false }, { "name": "__hash__", "qualname": "Request.__hash__", "full_name": "vllm_mlx.request.Request.__hash__", "kind": "method", "signature": "def __hash__(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "", "summary": "Method `Request.__hash__` calls `hash`; returns `hash(self.request_id)`.", "implementation": "Method `Request.__hash__` calls `hash`; returns `hash(self.request_id)`.", "documented": false, "public": false, "addressable": true, "line": 186, "end_line": 187, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L186-L187", "decorators": [], "calls": [ "hash" ], "state_reads": [ "self.request_id" ], "state_writes": [], "raises": [], "return_expressions": [ "hash(self.request_id)" ], "awaits": false, "yields": false }, { "name": "__eq__", "qualname": "Request.__eq__", "full_name": "vllm_mlx.request.Request.__eq__", "kind": "method", "signature": "def __eq__(self, other: object) -> bool", "parameters": [ { "name": "other", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "", "summary": "Method `Request.__eq__` calls `isinstance`; has 2 explicit return paths.", "implementation": "Method `Request.__eq__` calls `isinstance`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 189, "end_line": 192, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L189-L192", "decorators": [], "calls": [ "isinstance" ], "state_reads": [ "self.request_id" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "self.request_id == other.request_id" ], "awaits": false, "yields": false }, { "name": "RequestOutput", "qualname": "RequestOutput", "full_name": "vllm_mlx.request.RequestOutput", "kind": "class", "signature": "class RequestOutput", "parameters": [ { "name": "request_id", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "new_token_ids", "kind": "field", "annotation": "List[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "new_text", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "output_token_ids", "kind": "field", "annotation": "List[int]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "output_text", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." }, { "name": "finished", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "finish_reason", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "prompt_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "completion_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_drafts", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mtp_accepted", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "RequestOutput", "docstring": "Output for a single request after a generation step.\n\nThis is returned by the engine to communicate results back to the API layer.", "summary": "Output for a single request after a generation step.", "implementation": "Class `RequestOutput` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 196, "end_line": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L196-L227", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "usage", "qualname": "RequestOutput.usage", "full_name": "vllm_mlx.request.RequestOutput.usage", "kind": "method", "signature": "def usage(self) -> Dict[str, int]", "parameters": [], "return_annotation": "Dict[str, int]", "docstring": "Return usage statistics compatible with OpenAI API.", "summary": "Return usage statistics compatible with OpenAI API.", "implementation": "Method `RequestOutput.usage` returns `{'prompt_tokens': self.prompt_tokens, 'completion_tokens': self.completion_tokens, 'total_tokens': self.prompt_tokens +…`.", "documented": true, "public": true, "addressable": true, "line": 221, "end_line": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/request.py#L221-L227", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.prompt_tokens", "self.completion_tokens" ], "state_writes": [], "raises": [], "return_expressions": [ "{'prompt_tokens': self.prompt_tokens, 'completion_tokens': self.completion_tokens, 'total_tokens': self.prompt_tokens +…" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.rerank", "path": "vllm_mlx/rerank.py", "page_path": "reference/api/vllm_mlx/rerank.md", "docstring": "Reranker engine for cross-encoder models.\n\nProvides a dedicated RerankEngine with adapter-based scoring for the\nOpenAI/Jina-compatible /v1/rerank endpoint. Cross-encoder models use\nAutoModelForSequenceClassification-style loading, not mlx_lm.load.", "summary": "Reranker engine for cross-encoder models.", "line_count": 398, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L1-L398", "members": [ "logger", "RerankAdapter", "SigmoidAdapter", "_ADAPTER_REGISTRY", "get_adapter", "RerankEngine", "_build_classifier_model", "_MLXClassifierWrapper", "_ClassifierOutput" ], "symbols": [ { "name": "RerankAdapter", "qualname": "RerankAdapter", "full_name": "vllm_mlx.rerank.RerankAdapter", "kind": "class", "signature": "class RerankAdapter(ABC)", "parameters": [], "return_annotation": "RerankAdapter", "docstring": "Per-family adapter for reranker models.\n\nDifferent cross-encoder families use different tokenization patterns,\nscore extraction logic, and normalization functions. This contract\nisolates those differences so RerankEngine stays family-agnostic.", "summary": "Per-family adapter for reranker models.", "implementation": "Class `RerankAdapter` derives from `ABC` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 29, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L29-L80", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "tokenize_pair", "qualname": "RerankAdapter.tokenize_pair", "full_name": "vllm_mlx.rerank.RerankAdapter.tokenize_pair", "kind": "method", "signature": "def tokenize_pair(self, tokenizer, query: str, document: str, max_length: int) -> dict", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "The HuggingFace tokenizer instance." }, { "name": "query", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The query string." }, { "name": "document", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The document string." }, { "name": "max_length", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Truncation length (from the model's context window)." } ], "return_annotation": "dict", "docstring": "Tokenize a (query, document) pair for the cross-encoder.\n\nArgs:\n tokenizer: The HuggingFace tokenizer instance.\n query: The query string.\n document: The document string.\n max_length: Truncation length (from the model's context window).\n\nReturns:\n Dict with 'input_ids' and 'attention_mask' as numpy arrays.", "summary": "Tokenize a (query, document) pair for the cross-encoder.", "implementation": "Method `RerankAdapter.tokenize_pair` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 39, "end_line": 54, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L39-L54", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_score", "qualname": "RerankAdapter.extract_score", "full_name": "vllm_mlx.rerank.RerankAdapter.extract_score", "kind": "method", "signature": "def extract_score(self, logits) -> float", "parameters": [ { "name": "logits", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Model output logits (list or array), shape varies by model." } ], "return_annotation": "float", "docstring": "Extract a raw relevance score from model output logits.\n\nArgs:\n logits: Model output logits (list or array), shape varies by model.\n\nReturns:\n A single float raw score.", "summary": "Extract a raw relevance score from model output logits.", "implementation": "Method `RerankAdapter.extract_score` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 57, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L57-L67", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "normalize", "qualname": "RerankAdapter.normalize", "full_name": "vllm_mlx.rerank.RerankAdapter.normalize", "kind": "method", "signature": "def normalize(self, raw_score: float) -> float", "parameters": [ { "name": "raw_score", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "The raw score from extract_score()." } ], "return_annotation": "float", "docstring": "Normalize a raw score to [0, 1] range.\n\nArgs:\n raw_score: The raw score from extract_score().\n\nReturns:\n Normalized relevance score in [0, 1].", "summary": "Normalize a raw score to [0, 1] range.", "implementation": "Method `RerankAdapter.normalize` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L70-L80", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "SigmoidAdapter", "qualname": "SigmoidAdapter", "full_name": "vllm_mlx.rerank.SigmoidAdapter", "kind": "class", "signature": "class SigmoidAdapter(RerankAdapter)", "parameters": [], "return_annotation": "SigmoidAdapter", "docstring": "Default adapter for single-logit sigmoid rerankers.\n\nWorks with Jina Reranker v2, BGE Reranker v2, and MS-MARCO MiniLM\nfamilies. These models output a single relevance logit at position 0,\nnormalized via sigmoid.", "summary": "Default adapter for single-logit sigmoid rerankers.", "implementation": "Class `SigmoidAdapter` derives from `RerankAdapter` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 83, "end_line": 111, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L83-L111", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "tokenize_pair", "qualname": "SigmoidAdapter.tokenize_pair", "full_name": "vllm_mlx.rerank.SigmoidAdapter.tokenize_pair", "kind": "method", "signature": "def tokenize_pair(self, tokenizer, query: str, document: str, max_length: int) -> dict", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "query", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "document", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_length", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Tokenize as a sentence pair (query, document).", "summary": "Tokenize as a sentence pair (query, document).", "implementation": "Method `SigmoidAdapter.tokenize_pair` calls `tokenizer`; returns `tokenizer(query, document, padding=True, truncation=True, max_length=max_length, return_tensors='np')`.", "documented": true, "public": true, "addressable": true, "line": 92, "end_line": 103, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L92-L103", "decorators": [], "calls": [ "tokenizer" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tokenizer(query, document, padding=True, truncation=True, max_length=max_length, return_tensors='np')" ], "awaits": false, "yields": false }, { "name": "extract_score", "qualname": "SigmoidAdapter.extract_score", "full_name": "vllm_mlx.rerank.SigmoidAdapter.extract_score", "kind": "method", "signature": "def extract_score(self, logits) -> float", "parameters": [ { "name": "logits", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Extract the first logit as the relevance score.", "summary": "Extract the first logit as the relevance score.", "implementation": "Method `SigmoidAdapter.extract_score` calls `float`; returns `float(logits[0])`.", "documented": true, "public": true, "addressable": true, "line": 105, "end_line": 107, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L105-L107", "decorators": [], "calls": [ "float" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "float(logits[0])" ], "awaits": false, "yields": false }, { "name": "normalize", "qualname": "SigmoidAdapter.normalize", "full_name": "vllm_mlx.rerank.SigmoidAdapter.normalize", "kind": "method", "signature": "def normalize(self, raw_score: float) -> float", "parameters": [ { "name": "raw_score", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Apply sigmoid normalization.", "summary": "Apply sigmoid normalization.", "implementation": "Method `SigmoidAdapter.normalize` calls `math.exp`; returns `1.0 / (1.0 + math.exp(-raw_score))`.", "documented": true, "public": true, "addressable": true, "line": 109, "end_line": 111, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L109-L111", "decorators": [], "calls": [ "math.exp" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "1.0 / (1.0 + math.exp(-raw_score))" ], "awaits": false, "yields": false }, { "name": "get_adapter", "qualname": "get_adapter", "full_name": "vllm_mlx.rerank.get_adapter", "kind": "function", "signature": "def get_adapter(model_name: str) -> RerankAdapter", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "RerankAdapter", "docstring": "Return the appropriate adapter for a model.\n\nFalls back to SigmoidAdapter (works for Jina, BGE, MS-MARCO families).\nExtend _ADAPTER_REGISTRY for families that need different scoring.", "summary": "Return the appropriate adapter for a model.", "implementation": "Function `get_adapter` calls `_ADAPTER_REGISTRY['default']`; returns `_ADAPTER_REGISTRY['default']()`.", "documented": true, "public": true, "addressable": true, "line": 124, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L124-L133", "decorators": [], "calls": [ "_ADAPTER_REGISTRY['default']" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_ADAPTER_REGISTRY['default']()" ], "awaits": false, "yields": false }, { "name": "RerankEngine", "qualname": "RerankEngine", "full_name": "vllm_mlx.rerank.RerankEngine", "kind": "class", "signature": "class RerankEngine", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "token_budget", "kind": "positional or keyword", "annotation": "int", "default": "4096", "required": false, "description": "Optional positional or keyword input; defaults to `4096`." }, { "name": "max_concurrency", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Optional positional or keyword input; defaults to `1`." } ], "return_annotation": "RerankEngine", "docstring": "Reranker engine for cross-encoder sequence classification models.\n\nLoads cross-encoder models via transformers + MLX (safetensors weights).\nScores (query, document) pairs using the adapter contract for\nfamily-specific tokenization, score extraction, and normalization.\n\nSupports token-budget batching to avoid OOM on large document lists.", "summary": "Reranker engine for cross-encoder sequence classification models.", "implementation": "Class `RerankEngine` declares 6 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 136, "end_line": 338, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L136-L338", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "RerankEngine.__init__", "full_name": "vllm_mlx.rerank.RerankEngine.__init__", "kind": "method", "signature": "def __init__(self, model_name: str, token_budget: int=4096, max_concurrency: int=1)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "token_budget", "kind": "positional or keyword", "annotation": "int", "default": "4096", "required": false, "description": "Optional positional or keyword input; defaults to `4096`." }, { "name": "max_concurrency", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Optional positional or keyword input; defaults to `1`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `RerankEngine.__init__` updates `self.model_name`, `self.token_budget`, `self.max_concurrency`, `self._semaphore`; calls `asyncio.Semaphore`.", "implementation": "Method `RerankEngine.__init__` updates `self.model_name`, `self.token_budget`, `self.max_concurrency`, `self._semaphore`; calls `asyncio.Semaphore`.", "documented": false, "public": false, "addressable": true, "line": 147, "end_line": 159, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L147-L159", "decorators": [], "calls": [ "asyncio.Semaphore" ], "state_reads": [], "state_writes": [ "self.model_name", "self.token_budget", "self.max_concurrency", "self._semaphore", "self._model", "self._tokenizer", "self._adapter" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_loaded", "qualname": "RerankEngine.is_loaded", "full_name": "vllm_mlx.rerank.RerankEngine.is_loaded", "kind": "method", "signature": "def is_loaded(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return whether the reranking model has been loaded.", "summary": "Return whether the reranking model has been loaded.", "implementation": "Method `RerankEngine.is_loaded` returns `self._model is not None`.", "documented": true, "public": true, "addressable": true, "line": 162, "end_line": 165, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L162-L165", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._model" ], "state_writes": [], "raises": [], "return_expressions": [ "self._model is not None" ], "awaits": false, "yields": false }, { "name": "load", "qualname": "RerankEngine.load", "full_name": "vllm_mlx.rerank.RerankEngine.load", "kind": "method", "signature": "def load(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load the cross-encoder model and tokenizer.\n\nUses transformers AutoTokenizer and loads MLX weights from safetensors\nvia the model's from_pretrained or equivalent MLX loading path.", "summary": "Load the cross-encoder model and tokenizer.", "implementation": "Method `RerankEngine.load` updates `self._tokenizer`, `self._model`, `self._adapter`; calls `logger.info`, `time.perf_counter`, `AutoTokenizer.from_pretrained`, `self._load_mlx_model`.", "documented": true, "public": true, "addressable": true, "line": 167, "end_line": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L167-L184", "decorators": [], "calls": [ "logger.info", "time.perf_counter", "AutoTokenizer.from_pretrained", "self._load_mlx_model", "get_adapter" ], "state_reads": [ "self.model_name", "self._load_mlx_model" ], "state_writes": [ "self._tokenizer", "self._model", "self._adapter" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_load_mlx_model", "qualname": "RerankEngine._load_mlx_model", "full_name": "vllm_mlx.rerank.RerankEngine._load_mlx_model", "kind": "method", "signature": "def _load_mlx_model(model_name: str)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Load an MLX cross-encoder model from HuggingFace Hub.\n\nAttempts mlx-community weights first (safetensors), then falls back\nto transformers AutoModelForSequenceClassification with MLX conversion.", "summary": "Load an MLX cross-encoder model from HuggingFace Hub.", "implementation": "Method `RerankEngine._load_mlx_model` calls `snapshot_download`, `os.path.join`, `open`, `json.load`; can raise `FileNotFoundError`; returns `model`.", "documented": true, "public": false, "addressable": true, "line": 187, "end_line": 230, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L187-L230", "decorators": [ "staticmethod" ], "calls": [ "snapshot_download", "os.path.join", "open", "json.load", "glob.glob", "FileNotFoundError", "safe_open", "f.keys", "mx.array", "f.get_tensor", "config.get", "_build_classifier_model", "mx.eval", "model.parameters", "logger.error" ], "state_reads": [], "state_writes": [], "raises": [ "FileNotFoundError" ], "return_expressions": [ "model" ], "awaits": false, "yields": false }, { "name": "_ensure_loaded", "qualname": "RerankEngine._ensure_loaded", "full_name": "vllm_mlx.rerank.RerankEngine._ensure_loaded", "kind": "method", "signature": "def _ensure_loaded(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `RerankEngine._ensure_loaded` calls `self.load`.", "implementation": "Method `RerankEngine._ensure_loaded` calls `self.load`.", "documented": false, "public": false, "addressable": true, "line": 232, "end_line": 234, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L232-L234", "decorators": [], "calls": [ "self.load" ], "state_reads": [ "self.is_loaded", "self.load" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "score_pairs", "qualname": "RerankEngine.score_pairs", "full_name": "vllm_mlx.rerank.RerankEngine.score_pairs", "kind": "method", "signature": "def score_pairs(self, query: str, documents: list[str]) -> tuple[list[float], int]", "parameters": [ { "name": "query", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The query string." }, { "name": "documents", "kind": "positional or keyword", "annotation": "list[str]", "default": "", "required": true, "description": "List of document strings." } ], "return_annotation": "tuple[list[float], int]", "docstring": "Score each (query, document) pair and return normalized relevance scores.\n\nPairs are batched by token budget to control memory usage. Each batch\nis tokenized together and scored in a single forward pass.\nReturns (scores, total_tokens) where total_tokens reflects the\nactual tokenization used for scoring (consistent with adapter).\n\nArgs:\n query: The query string.\n documents: List of document strings.\n\nReturns:\n List of normalized relevance scores, one per document,\n in the same order as the input documents.", "summary": "Score each (query, document) pair and return normalized relevance scores.", "implementation": "Method `RerankEngine.score_pairs` calls `self._ensure_loaded`, `resolve_max_length`, `getattr`, `self._adapter.tokenize_pair`; returns `([score for _, score in all_scores], total_tokens)`.", "documented": true, "public": true, "addressable": true, "line": 236, "end_line": 338, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L236-L338", "decorators": [], "calls": [ "self._ensure_loaded", "resolve_max_length", "getattr", "self._adapter.tokenize_pair", "pair_encodings.append", "hasattr", "len", "pair_token_counts.append", "enumerate", "zip", "batches.append", "current_batch.append", "mx.array", "max", "raw_ids.tolist", "list", "raw_mask.tolist", "padded_ids.append", "padded_mask.append", "self._model", "output.logits.tolist", "self._adapter.extract_score", "self._adapter.normalize", "all_scores.append", "all_scores.sort", "sum" ], "state_reads": [ "self._ensure_loaded", "self._model", "self._tokenizer", "self._adapter.tokenize_pair", "self._adapter", "self.token_budget", "self._adapter.extract_score", "self._adapter.normalize" ], "state_writes": [], "raises": [], "return_expressions": [ "([score for _, score in all_scores], total_tokens)" ], "awaits": false, "yields": false }, { "name": "_build_classifier_model", "qualname": "_build_classifier_model", "full_name": "vllm_mlx.rerank._build_classifier_model", "kind": "function", "signature": "def _build_classifier_model(model_type, config, weights, num_labels)", "parameters": [ { "name": "model_type", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weights", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_labels", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Build an MLX sequence classification model from config and weights.\n\nThis is a thin wrapper that constructs the appropriate encoder\narchitecture with a classification head on top.", "summary": "Build an MLX sequence classification model from config and weights.", "implementation": "Function `_build_classifier_model` calls `_MLXClassifierWrapper`; returns `_MLXClassifierWrapper(config, weights, num_labels)`.", "documented": true, "public": false, "addressable": true, "line": 341, "end_line": 349, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L341-L349", "decorators": [], "calls": [ "_MLXClassifierWrapper" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_MLXClassifierWrapper(config, weights, num_labels)" ], "awaits": false, "yields": false }, { "name": "_MLXClassifierWrapper", "qualname": "_MLXClassifierWrapper", "full_name": "vllm_mlx.rerank._MLXClassifierWrapper", "kind": "class", "signature": "class _MLXClassifierWrapper", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weights", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_labels", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_MLXClassifierWrapper", "docstring": "Minimal MLX wrapper for sequence classification models.\n\nWraps loaded safetensors weights into a callable that returns\nlogits for (input_ids, attention_mask) pairs. Supports BERT-family\nand XLM-RoBERTa-family architectures commonly used as cross-encoders.", "summary": "Minimal MLX wrapper for sequence classification models.", "implementation": "Class `_MLXClassifierWrapper` declares 3 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 352, "end_line": 391, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L352-L391", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_MLXClassifierWrapper.__init__", "full_name": "vllm_mlx.rerank._MLXClassifierWrapper.__init__", "kind": "method", "signature": "def __init__(self, config: dict, weights: dict, num_labels: int)", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weights", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_labels", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_MLXClassifierWrapper.__init__` updates `self.config`, `self.weights`, `self.num_labels`, `self._params`; calls `list`, `weights.values`.", "implementation": "Method `_MLXClassifierWrapper.__init__` updates `self.config`, `self.weights`, `self.num_labels`, `self._params`; calls `list`, `weights.values`.", "documented": false, "public": false, "addressable": true, "line": 361, "end_line": 365, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L361-L365", "decorators": [], "calls": [ "list", "weights.values" ], "state_reads": [], "state_writes": [ "self.config", "self.weights", "self.num_labels", "self._params" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "parameters", "qualname": "_MLXClassifierWrapper.parameters", "full_name": "vllm_mlx.rerank._MLXClassifierWrapper.parameters", "kind": "method", "signature": "def parameters(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Return model parameters for mx.eval.", "summary": "Return model parameters for mx.eval.", "implementation": "Method `_MLXClassifierWrapper.parameters` returns `self._params`.", "documented": true, "public": false, "addressable": true, "line": 367, "end_line": 369, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L367-L369", "decorators": [], "calls": [], "state_reads": [ "self._params" ], "state_writes": [], "raises": [], "return_expressions": [ "self._params" ], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "_MLXClassifierWrapper.__call__", "full_name": "vllm_mlx.rerank._MLXClassifierWrapper.__call__", "kind": "method", "signature": "def __call__(self, input_ids: mx.array, attention_mask: mx.array=None)", "parameters": [ { "name": "input_ids", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attention_mask", "kind": "positional or keyword", "annotation": "mx.array", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Forward pass through the classifier.\n\nFor encoder-only cross-encoders, this runs the full transformer\nencoder and classification head. The exact layer wiring depends\non the model architecture.\n\nThis initial implementation uses a weight-lookup forward pass\nthat works for standard BERT/XLM-RoBERTa classifiers. For\nmodels with non-standard architectures, register a custom\nadapter via _ADAPTER_REGISTRY.", "summary": "Forward pass through the classifier.", "implementation": "Method `_MLXClassifierWrapper.__call__` calls `classifier_forward`, `_ClassifierOutput`; returns `_ClassifierOutput(logits=logits)`.", "documented": true, "public": false, "addressable": true, "line": 371, "end_line": 391, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L371-L391", "decorators": [], "calls": [ "classifier_forward", "_ClassifierOutput" ], "state_reads": [ "self.weights", "self.config" ], "state_writes": [], "raises": [], "return_expressions": [ "_ClassifierOutput(logits=logits)" ], "awaits": false, "yields": false }, { "name": "_ClassifierOutput", "qualname": "_ClassifierOutput", "full_name": "vllm_mlx.rerank._ClassifierOutput", "kind": "class", "signature": "class _ClassifierOutput", "parameters": [ { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_ClassifierOutput", "docstring": "Simple container for classifier output logits.", "summary": "Simple container for classifier output logits.", "implementation": "Class `_ClassifierOutput` declares 1 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 394, "end_line": 398, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L394-L398", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_ClassifierOutput.__init__", "full_name": "vllm_mlx.rerank._ClassifierOutput.__init__", "kind": "method", "signature": "def __init__(self, logits: mx.array)", "parameters": [ { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_ClassifierOutput.__init__` updates `self.logits`.", "implementation": "Method `_ClassifierOutput.__init__` updates `self.logits`.", "documented": false, "public": false, "addressable": true, "line": 397, "end_line": 398, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank.py#L397-L398", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.logits" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.rerank_forward", "path": "vllm_mlx/rerank_forward.py", "page_path": "reference/api/vllm_mlx/rerank_forward.md", "docstring": "MLX forward pass for BERT-family sequence classification models.\n\nImplements a from-weights forward pass for cross-encoder rerankers\nthat use the standard BERT/XLM-RoBERTa architecture with a\nclassification head. This avoids pulling in the full transformers\nmodeling stack at inference time — only the tokenizer is needed\nfrom transformers.", "summary": "MLX forward pass for BERT-family sequence classification models.", "line_count": 265, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L1-L265", "members": [ "classifier_forward", "_position_ids_for_config", "_classification_head_forward", "_detect_prefix", "_layer_norm", "_encoder_layer", "_gelu", "_gelu_new", "_relu", "_silu", "_apply_hidden_activation" ], "symbols": [ { "name": "classifier_forward", "qualname": "classifier_forward", "full_name": "vllm_mlx.rerank_forward.classifier_forward", "kind": "function", "signature": "def classifier_forward(input_ids: mx.array, attention_mask: mx.array, weights: dict[str, mx.array], config: dict) -> mx.array", "parameters": [ { "name": "input_ids", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "(batch, seq_len) token IDs." }, { "name": "attention_mask", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "(batch, seq_len) attention mask (1=attend, 0=pad)." }, { "name": "weights", "kind": "positional or keyword", "annotation": "dict[str, mx.array]", "default": "", "required": true, "description": "Dict mapping weight name -> mx.array." }, { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Model config dict (from config.json)." } ], "return_annotation": "mx.array", "docstring": "Run a BERT-family classifier forward pass on MLX.\n\nArgs:\n input_ids: (batch, seq_len) token IDs.\n attention_mask: (batch, seq_len) attention mask (1=attend, 0=pad).\n weights: Dict mapping weight name -> mx.array.\n config: Model config dict (from config.json).\n\nReturns:\n logits: (batch, num_labels) classification logits.", "summary": "Run a BERT-family classifier forward pass on MLX.", "implementation": "Function `classifier_forward` calls `config.get`, `_detect_prefix`, `_position_ids_for_config`, `mx.zeros_like`; returns `logits`.", "documented": true, "public": true, "addressable": true, "line": 16, "end_line": 85, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L16-L85", "decorators": [], "calls": [ "config.get", "_detect_prefix", "_position_ids_for_config", "mx.zeros_like", "_layer_norm", "attention_mask[:, None, None, :].astype", "range", "_encoder_layer", "weights.get", "mx.tanh", "_classification_head_forward" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "logits" ], "awaits": false, "yields": false }, { "name": "_position_ids_for_config", "qualname": "_position_ids_for_config", "full_name": "vllm_mlx.rerank_forward._position_ids_for_config", "kind": "function", "signature": "def _position_ids_for_config(config: dict, input_ids: mx.array, attention_mask: mx.array | None) -> mx.array", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "input_ids", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attention_mask", "kind": "positional or keyword", "annotation": "mx.array | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Build BERT or RoBERTa-family absolute position IDs.", "summary": "Build BERT or RoBERTa-family absolute position IDs.", "implementation": "Function `_position_ids_for_config` calls `str(config.get('model_type', '')).lower`, `str`, `config.get`, `mx.arange`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 88, "end_line": 105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L88-L105", "decorators": [], "calls": [ "str(config.get('model_type', '')).lower", "str", "config.get", "mx.arange", "int", "attention_mask.astype", "mx.cumsum", "positions.astype" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.arange(seq_len)[None, :]", "mx.arange(padding_idx + 1, seq_len + padding_idx + 1)[None, :]", "positions.astype(mx.int32)" ], "awaits": false, "yields": false }, { "name": "_classification_head_forward", "qualname": "_classification_head_forward", "full_name": "vllm_mlx.rerank_forward._classification_head_forward", "kind": "function", "signature": "def _classification_head_forward(pooled: mx.array, weights: dict[str, mx.array]) -> mx.array", "parameters": [ { "name": "pooled", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weights", "kind": "positional or keyword", "annotation": "dict[str, mx.array]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Run BERT flat or XLM-RoBERTa two-layer sequence-classification head.", "summary": "Run BERT flat or XLM-RoBERTa two-layer sequence-classification head.", "implementation": "Function `_classification_head_forward` calls `mx.tanh`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 108, "end_line": 122, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L108-L122", "decorators": [], "calls": [ "mx.tanh" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hidden @ weights['classifier.out_proj.weight'].T + weights['classifier.out_proj.bias']", "pooled @ weights['classifier.weight'].T + weights['classifier.bias']" ], "awaits": false, "yields": false }, { "name": "_detect_prefix", "qualname": "_detect_prefix", "full_name": "vllm_mlx.rerank_forward._detect_prefix", "kind": "function", "signature": "def _detect_prefix(weights: dict) -> str", "parameters": [ { "name": "weights", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Detect the model weight prefix (bert, roberta, xlm-roberta).", "summary": "Detect the model weight prefix (bert, roberta, xlm-roberta).", "implementation": "Function `_detect_prefix` calls `key.startswith`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 125, "end_line": 135, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L125-L135", "decorators": [], "calls": [ "key.startswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'bert'", "'roberta'", "'xlm-roberta'" ], "awaits": false, "yields": false }, { "name": "_layer_norm", "qualname": "_layer_norm", "full_name": "vllm_mlx.rerank_forward._layer_norm", "kind": "function", "signature": "def _layer_norm(x: mx.array, weight: mx.array, bias: mx.array, eps: float) -> mx.array", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weight", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "bias", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "eps", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Apply layer normalization.", "summary": "Apply layer normalization.", "implementation": "Function `_layer_norm` calls `mx.mean`, `mx.var`, `mx.sqrt`; returns `weight * (x - mean) / mx.sqrt(var + eps) + bias`.", "documented": true, "public": false, "addressable": true, "line": 138, "end_line": 142, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L138-L142", "decorators": [], "calls": [ "mx.mean", "mx.var", "mx.sqrt" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "weight * (x - mean) / mx.sqrt(var + eps) + bias" ], "awaits": false, "yields": false }, { "name": "_encoder_layer", "qualname": "_encoder_layer", "full_name": "vllm_mlx.rerank_forward._encoder_layer", "kind": "function", "signature": "def _encoder_layer(hidden: mx.array, ext_mask: mx.array | None, weights: dict, prefix: str, num_heads: int, head_dim: int, eps: float, config: dict) -> mx.array", "parameters": [ { "name": "hidden", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "ext_mask", "kind": "positional or keyword", "annotation": "mx.array | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "weights", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prefix", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_heads", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "head_dim", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "eps", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Run one BERT encoder layer (self-attention + FFN).", "summary": "Run one BERT encoder layer (self-attention + FFN).", "implementation": "Function `_encoder_layer` calls `(hidden @ q_w.T + q_b).reshape(batch_size, seq_len, num_heads, head_dim).transpose`, `(hidden @ q_w.T + q_b).reshape`, `(hidden @ k_w.T + k_b).reshape(batch_size, seq_len, num_heads, head_dim).transpose`, `(hidden @ k_w.T + k_b).reshape`; returns `hidden`.", "documented": true, "public": false, "addressable": true, "line": 145, "end_line": 217, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L145-L217", "decorators": [], "calls": [ "(hidden @ q_w.T + q_b).reshape(batch_size, seq_len, num_heads, head_dim).transpose", "(hidden @ q_w.T + q_b).reshape", "(hidden @ k_w.T + k_b).reshape(batch_size, seq_len, num_heads, head_dim).transpose", "(hidden @ k_w.T + k_b).reshape", "(hidden @ v_w.T + v_b).reshape(batch_size, seq_len, num_heads, head_dim).transpose", "(hidden @ v_w.T + v_b).reshape", "k.transpose", "mx.softmax", "(attn_probs @ v).transpose(0, 2, 1, 3).reshape", "(attn_probs @ v).transpose", "_layer_norm", "_apply_hidden_activation" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hidden" ], "awaits": false, "yields": false }, { "name": "_gelu", "qualname": "_gelu", "full_name": "vllm_mlx.rerank_forward._gelu", "kind": "function", "signature": "def _gelu(x: mx.array) -> mx.array", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "GELU activation (exact form).", "summary": "GELU activation (exact form).", "implementation": "Function `_gelu` calls `nn.gelu`; returns `nn.gelu(x)`.", "documented": true, "public": false, "addressable": true, "line": 220, "end_line": 222, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L220-L222", "decorators": [], "calls": [ "nn.gelu" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "nn.gelu(x)" ], "awaits": false, "yields": false }, { "name": "_gelu_new", "qualname": "_gelu_new", "full_name": "vllm_mlx.rerank_forward._gelu_new", "kind": "function", "signature": "def _gelu_new(x: mx.array) -> mx.array", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "BERT GELU approximation used by transformers gelu_new.", "summary": "BERT GELU approximation used by transformers gelu_new.", "implementation": "Function `_gelu_new` calls `mx.tanh`; returns `0.5 * x * (1.0 + mx.tanh(0.7978845608028654 * (x + 0.044715 * x ** 3)))`.", "documented": true, "public": false, "addressable": true, "line": 225, "end_line": 227, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L225-L227", "decorators": [], "calls": [ "mx.tanh" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "0.5 * x * (1.0 + mx.tanh(0.7978845608028654 * (x + 0.044715 * x ** 3)))" ], "awaits": false, "yields": false }, { "name": "_relu", "qualname": "_relu", "full_name": "vllm_mlx.rerank_forward._relu", "kind": "function", "signature": "def _relu(x: mx.array) -> mx.array", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "ReLU activation.", "summary": "ReLU activation.", "implementation": "Function `_relu` calls `mx.maximum`; returns `mx.maximum(x, 0)`.", "documented": true, "public": false, "addressable": true, "line": 230, "end_line": 232, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L230-L232", "decorators": [], "calls": [ "mx.maximum" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.maximum(x, 0)" ], "awaits": false, "yields": false }, { "name": "_silu", "qualname": "_silu", "full_name": "vllm_mlx.rerank_forward._silu", "kind": "function", "signature": "def _silu(x: mx.array) -> mx.array", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "SiLU/swish activation.", "summary": "SiLU/swish activation.", "implementation": "Function `_silu` calls `mx.sigmoid`; returns `x * mx.sigmoid(x)`.", "documented": true, "public": false, "addressable": true, "line": 235, "end_line": 237, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L235-L237", "decorators": [], "calls": [ "mx.sigmoid" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "x * mx.sigmoid(x)" ], "awaits": false, "yields": false }, { "name": "_apply_hidden_activation", "qualname": "_apply_hidden_activation", "full_name": "vllm_mlx.rerank_forward._apply_hidden_activation", "kind": "function", "signature": "def _apply_hidden_activation(x: mx.array, config: dict) -> mx.array", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "mx.array", "docstring": "Apply the configured encoder hidden activation.\n\nThe MLX reranker forward pass targets standard BERT/XLM-RoBERTa-style\nsequence classifiers. Configs that request an activation outside that\nsupported contract fail explicitly instead of silently using GELU.", "summary": "Apply the configured encoder hidden activation.", "implementation": "Function `_apply_hidden_activation` calls `config.get`, `isinstance`, `hidden_act.get`, `str(hidden_act).lower`; can raise `ValueError`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 240, "end_line": 265, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/rerank_forward.py#L240-L265", "decorators": [], "calls": [ "config.get", "isinstance", "hidden_act.get", "str(hidden_act).lower", "str", "_gelu", "_gelu_new", "_relu", "_silu", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "_gelu(x)", "_gelu_new(x)", "_relu(x)", "_silu(x)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.scheduler", "path": "vllm_mlx/scheduler.py", "page_path": "reference/api/vllm_mlx/scheduler.md", "docstring": "Scheduler for vllm-mlx continuous batching.\n\nThis module provides a Scheduler class that manages request scheduling\nusing mlx-lm's BatchGenerator for efficient continuous batching.\n\nThe scheduler follows vLLM's design with:\n- Waiting queue for pending requests\n- Running set for active requests\n- Continuous batching via BatchGenerator", "summary": "Scheduler for vllm-mlx continuous batching.", "line_count": 3518, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1-L3518", "members": [ "logger", "CACHE_CORRUPTION_PATTERNS", "_normalize_logits_processors", "_sanitize_batch_generator_logits_processors", "SchedulingPolicy", "SchedulerConfig", "SchedulerOutput", "_install_prompt_cache_save", "_install_chunked_prefill", "_MTPStatsState", "_configure_chunked_prefill", "_install_mtp", "_mtp_status_snapshot", "Scheduler" ], "symbols": [ { "name": "_normalize_logits_processors", "qualname": "_normalize_logits_processors", "full_name": "vllm_mlx.scheduler._normalize_logits_processors", "kind": "function", "signature": "def _normalize_logits_processors(logits_processors)", "parameters": [ { "name": "logits_processors", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Normalize empty per-sequence processor slots to lists.", "summary": "Normalize empty per-sequence processor slots to lists.", "implementation": "Function `_normalize_logits_processors` has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 46, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L46-L50", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "[processors or [] for processors in logits_processors]" ], "awaits": false, "yields": false }, { "name": "_sanitize_batch_generator_logits_processors", "qualname": "_sanitize_batch_generator_logits_processors", "full_name": "vllm_mlx.scheduler._sanitize_batch_generator_logits_processors", "kind": "function", "signature": "def _sanitize_batch_generator_logits_processors(batch_generator) -> None", "parameters": [ { "name": "batch_generator", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Sanitize stale BatchGenerator processor state before decode.", "summary": "Sanitize stale BatchGenerator processor state before decode.", "implementation": "Function `_sanitize_batch_generator_logits_processors` calls `getattr`, `hasattr`, `_normalize_logits_processors`, `isinstance`.", "documented": true, "public": false, "addressable": true, "line": 53, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L53-L65", "decorators": [], "calls": [ "getattr", "hasattr", "_normalize_logits_processors", "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "SchedulingPolicy", "qualname": "SchedulingPolicy", "full_name": "vllm_mlx.scheduler.SchedulingPolicy", "kind": "class", "signature": "class SchedulingPolicy(Enum)", "parameters": [], "return_annotation": "SchedulingPolicy", "docstring": "Scheduling policy for request ordering.", "summary": "Scheduling policy for request ordering.", "implementation": "Class `SchedulingPolicy` derives from `Enum` and declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 68, "end_line": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L68-L72", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "SchedulerConfig", "qualname": "SchedulerConfig", "full_name": "vllm_mlx.scheduler.SchedulerConfig", "kind": "class", "signature": "class SchedulerConfig", "parameters": [ { "name": "max_num_seqs", "kind": "field", "annotation": "int", "default": "256", "required": false, "description": "Optional constructor field; defaults to `256`." }, { "name": "max_num_batched_tokens", "kind": "field", "annotation": "int", "default": "8192", "required": false, "description": "Optional constructor field; defaults to `8192`." }, { "name": "policy", "kind": "field", "annotation": "SchedulingPolicy", "default": "SchedulingPolicy.FCFS", "required": false, "description": "Optional constructor field; defaults to `SchedulingPolicy.FCFS`." }, { "name": "prefill_batch_size", "kind": "field", "annotation": "int", "default": "8", "required": false, "description": "Optional constructor field; defaults to `8`." }, { "name": "completion_batch_size", "kind": "field", "annotation": "int", "default": "32", "required": false, "description": "Optional constructor field; defaults to `32`." }, { "name": "prefill_step_size", "kind": "field", "annotation": "int", "default": "2048", "required": false, "description": "Optional constructor field; defaults to `2048`." }, { "name": "mllm_prefill_step_size", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "enable_prefix_cache", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "prefix_cache_size", "kind": "field", "annotation": "int", "default": "100", "required": false, "description": "Optional constructor field; defaults to `100`." }, { "name": "use_memory_aware_cache", "kind": "field", "annotation": "bool", "default": "True", "required": false, "description": "Optional constructor field; defaults to `True`." }, { "name": "cache_memory_mb", "kind": "field", "annotation": "Optional[int]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "cache_memory_percent", "kind": "field", "annotation": "float", "default": "0.2", "required": false, "description": "Optional constructor field; defaults to `0.2`." }, { "name": "kv_cache_quantization", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "kv_cache_quantization_bits", "kind": "field", "annotation": "int", "default": "8", "required": false, "description": "Optional constructor field; defaults to `8`." }, { "name": "kv_cache_quantization_group_size", "kind": "field", "annotation": "int", "default": "64", "required": false, "description": "Optional constructor field; defaults to `64`." }, { "name": "kv_cache_min_quantize_tokens", "kind": "field", "annotation": "int", "default": "256", "required": false, "description": "Optional constructor field; defaults to `256`." }, { "name": "use_paged_cache", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "paged_cache_block_size", "kind": "field", "annotation": "int", "default": "64", "required": false, "description": "Optional constructor field; defaults to `64`." }, { "name": "max_cache_blocks", "kind": "field", "annotation": "int", "default": "1000", "required": false, "description": "Optional constructor field; defaults to `1000`." }, { "name": "chunked_prefill_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "mid_prefill_save_interval", "kind": "field", "annotation": "int", "default": "8192", "required": false, "description": "Optional constructor field; defaults to `8192`." }, { "name": "ssd_cache_dir", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "ssd_cache_max_gb", "kind": "field", "annotation": "float", "default": "10.0", "required": false, "description": "Optional constructor field; defaults to `10.0`." }, { "name": "max_kv_size", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "enable_mtp", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." }, { "name": "mtp_num_draft_tokens", "kind": "field", "annotation": "int", "default": "1", "required": false, "description": "Optional constructor field; defaults to `1`." }, { "name": "mtp_optimistic", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "SchedulerConfig", "docstring": "Configuration for the scheduler.", "summary": "Configuration for the scheduler.", "implementation": "Class `SchedulerConfig` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 76, "end_line": 140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L76-L140", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__post_init__", "qualname": "SchedulerConfig.__post_init__", "full_name": "vllm_mlx.scheduler.SchedulerConfig.__post_init__", "kind": "method", "signature": "def __post_init__(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `SchedulerConfig.__post_init__` calls `ValueError`; can raise `ValueError`.", "implementation": "Method `SchedulerConfig.__post_init__` calls `ValueError`; can raise `ValueError`.", "documented": false, "public": false, "addressable": true, "line": 138, "end_line": 140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L138-L140", "decorators": [], "calls": [ "ValueError" ], "state_reads": [ "self.mllm_prefill_step_size" ], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "SchedulerOutput", "qualname": "SchedulerOutput", "full_name": "vllm_mlx.scheduler.SchedulerOutput", "kind": "class", "signature": "class SchedulerOutput", "parameters": [ { "name": "scheduled_request_ids", "kind": "field", "annotation": "List[str]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "num_scheduled_tokens", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "finished_request_ids", "kind": "field", "annotation": "Set[str]", "default": "field(default_factory=set)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=set)`." }, { "name": "outputs", "kind": "field", "annotation": "List[RequestOutput]", "default": "field(default_factory=list)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=list)`." }, { "name": "has_work", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "SchedulerOutput", "docstring": "Output from a scheduling step.\n\nContains information about what was scheduled and results.", "summary": "Output from a scheduling step.", "implementation": "Class `SchedulerOutput` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 144, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L144-L160", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_install_prompt_cache_save", "qualname": "_install_prompt_cache_save", "full_name": "vllm_mlx.scheduler._install_prompt_cache_save", "kind": "function", "signature": "def _install_prompt_cache_save(batch_gen: 'BatchGenerator', prompt_cache_save) -> None", "parameters": [ { "name": "batch_gen", "kind": "positional or keyword", "annotation": "'BatchGenerator'", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_cache_save", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Monkey-patch ``_process_prompts`` to capture prompt-only cache state.\n\nCan be installed independently of chunked prefill. If chunked prefill is\nalso installed, *it* takes over ``_process_prompts`` and invokes the\ncallback itself, so call this **before** ``_install_chunked_prefill``.", "summary": "Monkey-patch ``_process_prompts`` to capture prompt-only cache state.", "implementation": "Function `_install_prompt_cache_save` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": false, "addressable": true, "line": 163, "end_line": 187, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L163-L187", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_patched_process_prompts", "qualname": "_install_prompt_cache_save._patched_process_prompts", "full_name": "vllm_mlx.scheduler._install_prompt_cache_save._patched_process_prompts", "kind": "nested function", "signature": "def _patched_process_prompts(prompts, _self=batch_gen)", "parameters": [ { "name": "prompts", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "_self", "kind": "positional or keyword", "annotation": "not annotated", "default": "batch_gen", "required": false, "description": "Optional positional or keyword input; defaults to `batch_gen`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_prompt_cache_save._patched_process_prompts` calls `_orig_process_prompts`, `enumerate`, `prompt_cache_save`, `batch.extract_cache`; returns `batch`.", "implementation": "Nested Function `_install_prompt_cache_save._patched_process_prompts` calls `_orig_process_prompts`, `enumerate`, `prompt_cache_save`, `batch.extract_cache`; returns `batch`.", "documented": false, "public": false, "addressable": false, "line": 177, "end_line": 185, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L177-L185", "decorators": [], "calls": [ "_orig_process_prompts", "enumerate", "prompt_cache_save", "batch.extract_cache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "batch" ], "awaits": false, "yields": false }, { "name": "_install_chunked_prefill", "qualname": "_install_chunked_prefill", "full_name": "vllm_mlx.scheduler._install_chunked_prefill", "kind": "function", "signature": "def _install_chunked_prefill(batch_gen: 'BatchGenerator', budget: int, mid_prefill_save=None, prompt_cache_save=None, pending_abort_ids: Optional[Set[str]]=None, uid_to_request_id: Optional[Dict[int, str]]=None, requests: Optional[Dict[str, Any]]=None) -> None", "parameters": [ { "name": "batch_gen", "kind": "positional or keyword", "annotation": "'BatchGenerator'", "default": "", "required": true, "description": "The BatchGenerator to patch." }, { "name": "budget", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Max tokens per prefill chunk." }, { "name": "mid_prefill_save", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional callback(uid, processed, prompt_cache) called after each chunk to save intermediate KV cache state." }, { "name": "prompt_cache_save", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "pending_abort_ids", "kind": "positional or keyword", "annotation": "Optional[Set[str]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "uid_to_request_id", "kind": "positional or keyword", "annotation": "Optional[Dict[int, str]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "requests", "kind": "positional or keyword", "annotation": "Optional[Dict[str, Any]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Monkey-patch a BatchGenerator instance so that large prefills are\nbroken into chunks of at most *budget* tokens each.\n\nBetween chunks the generation loop gets a chance to produce one token\nfor every active request, preventing starvation during long prefills.\n\nArgs:\n batch_gen: The BatchGenerator to patch.\n budget: Max tokens per prefill chunk.\n mid_prefill_save: Optional callback(uid, processed, prompt_cache)\n called after each chunk to save intermediate KV cache state.", "summary": "Monkey-patch a BatchGenerator instance so that large prefills are broken into chunks of at most *budget* tokens each.", "implementation": "Function `_install_chunked_prefill` calls `logger.info`.", "documented": true, "public": false, "addressable": true, "line": 190, "end_line": 697, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L190-L697", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_lazy_extract_cache", "qualname": "_install_chunked_prefill._lazy_extract_cache", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._lazy_extract_cache", "kind": "nested function", "signature": "def _lazy_extract_cache(cache, idx)", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "idx", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_chunked_prefill._lazy_extract_cache` calls `c.extract`; returns `(c.extract(idx) for c in cache)`.", "implementation": "Nested Function `_install_chunked_prefill._lazy_extract_cache` calls `c.extract`; returns `(c.extract(idx) for c in cache)`.", "documented": false, "public": false, "addressable": false, "line": 225, "end_line": 226, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L225-L226", "decorators": [], "calls": [ "c.extract" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(c.extract(idx) for c in cache)" ], "awaits": false, "yields": false }, { "name": "_batch_cls", "qualname": "_install_chunked_prefill._batch_cls", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._batch_cls", "kind": "nested class", "signature": "class _batch_cls", "parameters": [ { "name": "uids", "kind": "field", "annotation": "List[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "y", "kind": "field", "annotation": "Any", "default": "", "required": true, "description": "Required constructor field." }, { "name": "logprobs", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "max_tokens", "kind": "field", "annotation": "List[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "num_tokens", "kind": "field", "annotation": "List[int]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "cache", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "samplers", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "logits_processors", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tokens", "kind": "field", "annotation": "List[Any]", "default": "", "required": true, "description": "Required constructor field." } ], "return_annotation": "_batch_cls", "docstring": "", "summary": "Nested Class `_install_chunked_prefill._batch_cls` declares 4 direct member(s).", "implementation": "Nested Class `_install_chunked_prefill._batch_cls` declares 4 direct member(s).", "documented": false, "public": false, "addressable": false, "line": 233, "end_line": 273, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L233-L273", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__len__", "qualname": "_install_chunked_prefill._batch_cls.__len__", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._batch_cls.__len__", "kind": "nested function", "signature": "def __len__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_chunked_prefill._batch_cls.__len__` calls `len`; returns `len(self.uids)`.", "implementation": "Nested Function `_install_chunked_prefill._batch_cls.__len__` calls `len`; returns `len(self.uids)`.", "documented": false, "public": false, "addressable": false, "line": 244, "end_line": 245, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L244-L245", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.uids" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.uids)" ], "awaits": false, "yields": false }, { "name": "filter", "qualname": "_install_chunked_prefill._batch_cls.filter", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._batch_cls.filter", "kind": "nested function", "signature": "def filter(self, keep_idx: List[int])", "parameters": [ { "name": "keep_idx", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_chunked_prefill._batch_cls.filter` updates `self.uids`, `self.logprobs`, `self.max_tokens`, `self.num_tokens`; calls `mx.array`, `c.filter`.", "implementation": "Nested Function `_install_chunked_prefill._batch_cls.filter` updates `self.uids`, `self.logprobs`, `self.max_tokens`, `self.num_tokens`; calls `mx.array`, `c.filter`.", "documented": false, "public": false, "addressable": false, "line": 247, "end_line": 258, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L247-L258", "decorators": [], "calls": [ "mx.array", "c.filter" ], "state_reads": [ "self.uids", "self.logprobs", "self.max_tokens", "self.num_tokens", "self.samplers", "self.logits_processors", "self.tokens", "self.y", "self.cache" ], "state_writes": [ "self.uids", "self.logprobs", "self.max_tokens", "self.num_tokens", "self.samplers", "self.logits_processors", "self.tokens", "self.y" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extend", "qualname": "_install_chunked_prefill._batch_cls.extend", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._batch_cls.extend", "kind": "nested function", "signature": "def extend(self, other)", "parameters": [ { "name": "other", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_chunked_prefill._batch_cls.extend` updates `self.y`; calls `self.uids.extend`, `mx.concatenate`, `self.logprobs.extend`, `self.num_tokens.extend`.", "implementation": "Nested Function `_install_chunked_prefill._batch_cls.extend` updates `self.y`; calls `self.uids.extend`, `mx.concatenate`, `self.logprobs.extend`, `self.num_tokens.extend`.", "documented": false, "public": false, "addressable": false, "line": 260, "end_line": 270, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L260-L270", "decorators": [], "calls": [ "self.uids.extend", "mx.concatenate", "self.logprobs.extend", "self.num_tokens.extend", "self.max_tokens.extend", "self.samplers.extend", "self.logits_processors.extend", "self.tokens.extend", "zip", "c.extend" ], "state_reads": [ "self.uids.extend", "self.uids", "self.y", "self.logprobs.extend", "self.logprobs", "self.num_tokens.extend", "self.num_tokens", "self.max_tokens.extend", "self.max_tokens", "self.samplers.extend", "self.samplers", "self.logits_processors.extend", "self.logits_processors", "self.tokens.extend", "self.tokens", "self.cache" ], "state_writes": [ "self.y" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_cache", "qualname": "_install_chunked_prefill._batch_cls.extract_cache", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._batch_cls.extract_cache", "kind": "nested function", "signature": "def extract_cache(self, idx)", "parameters": [ { "name": "idx", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_chunked_prefill._batch_cls.extract_cache` calls `c.extract`; returns `[c.extract(idx) for c in self.cache]`.", "implementation": "Nested Function `_install_chunked_prefill._batch_cls.extract_cache` calls `c.extract`; returns `[c.extract(idx) for c in self.cache]`.", "documented": false, "public": false, "addressable": false, "line": 272, "end_line": 273, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L272-L273", "decorators": [], "calls": [ "c.extract" ], "state_reads": [ "self.cache" ], "state_writes": [], "raises": [], "return_expressions": [ "[c.extract(idx) for c in self.cache]" ], "awaits": false, "yields": false }, { "name": "_patched_process_prompts", "qualname": "_install_chunked_prefill._patched_process_prompts", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._patched_process_prompts", "kind": "nested function", "signature": "def _patched_process_prompts(prompts, _self=batch_gen)", "parameters": [ { "name": "prompts", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "_self", "kind": "positional or keyword", "annotation": "not annotated", "default": "batch_gen", "required": false, "description": "Optional positional or keyword input; defaults to `batch_gen`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_chunked_prefill._patched_process_prompts` calls `_orig_process_prompts`, `enumerate`, `prompt_cache_save`, `batch.extract_cache`; returns `batch`.", "implementation": "Nested Function `_install_chunked_prefill._patched_process_prompts` calls `_orig_process_prompts`, `enumerate`, `prompt_cache_save`, `batch.extract_cache`; returns `batch`.", "documented": false, "public": false, "addressable": false, "line": 291, "end_line": 299, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L291-L299", "decorators": [], "calls": [ "_orig_process_prompts", "enumerate", "prompt_cache_save", "batch.extract_cache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "batch" ], "awaits": false, "yields": false }, { "name": "_generation_step", "qualname": "_install_chunked_prefill._generation_step", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._generation_step", "kind": "nested function", "signature": "def _generation_step(self=batch_gen)", "parameters": [], "return_annotation": "not annotated", "docstring": "Run one generation step on the active batch. Returns responses.", "summary": "Run one generation step on the active batch.", "implementation": "Nested Function `_install_chunked_prefill._generation_step` updates `self._stats.generation_time`, `self.active_batch`, `self._stats.generation_tokens`; calls `len`, `_time.perf_counter`, `enumerate`, `mx.concatenate`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 303, "end_line": 360, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L303-L360", "decorators": [], "calls": [ "len", "_time.perf_counter", "enumerate", "mx.concatenate", "self._step", "mx.async_eval", "y.tolist", "zip", "end_idx.append", "keep_idx.append", "batch.extract_cache", "responses.append", "self.Response", "batch.filter" ], "state_reads": [ "self.active_batch", "self._step", "self._stats", "self.stop_tokens", "self.Response" ], "state_writes": [ "self._stats.generation_time", "self.active_batch", "self._stats.generation_tokens" ], "raises": [], "return_expressions": [ "[]", "responses" ], "awaits": false, "yields": false }, { "name": "_chunked_next", "qualname": "_install_chunked_prefill._chunked_next", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._chunked_next", "kind": "nested function", "signature": "def _chunked_next(self=batch_gen)", "parameters": [], "return_annotation": "not annotated", "docstring": "Replacement for _next() that chunks large prefills.\n\nOnly intercepts when:\n1. A partial prefill is in progress (_partial is not None)\n2. The next prompt batch exceeds the budget\n\nEverything else delegates to the original _next().", "summary": "Replacement for _next() that chunks large prefills.", "implementation": "Nested Function `_install_chunked_prefill._chunked_next` updates `self._partial`, `self.active_batch`, `self._stats.prompt_time`, `self._stats.generation_time`; calls `uid_to_request_id.get`, `logger.info`, `mx.clear_cache`, `self._generation_step`; returns `self._generation_step()`.", "documented": true, "public": false, "addressable": false, "line": 362, "end_line": 678, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L362-L678", "decorators": [], "calls": [ "uid_to_request_id.get", "logger.info", "mx.clear_cache", "self._generation_step", "_time.perf_counter", "max", "int", "partial.get", "min", "self.model", "mx.contiguous", "mx.eval", "self.prompt_progress_callback", "len", "mid_prefill_save", "c.finalize", "self.prompt_checkpoint_callback", "_lazy_extract_cache", "enumerate", "self._step", "mx.async_eval", "_batch_cls", "list", "prompt_cache_save", "new_batch.extract_cache", "self.active_batch.extend", "sum", "requests.get", "getattr", "zip", "mx.array", "all", "c[0].empty", "_left_pad_prompts", "_make_cache", "_right_pad_prompts", "_merge_caches", "c.prepare", "self._process_prompts" ], "state_reads": [ "self._partial", "self._generation_step", "self.model", "self.prompt_progress_callback", "self.prompt_checkpoint_callback", "self._step", "self.active_batch", "self.active_batch.extend", "self._stats", "self.completion_batch_size", "self.prefill_batch_size", "self.unprocessed_prompts", "self.active_batch.y", "self.active_batch.logprobs", "self.max_kv_size", "self._process_prompts" ], "state_writes": [ "self._partial", "self.active_batch", "self._stats.prompt_time", "self._stats.generation_time", "self._stats.prompt_tokens", "self.unprocessed_prompts" ], "raises": [], "return_expressions": [ "self._generation_step()" ], "awaits": false, "yields": false }, { "name": "_patched_remove", "qualname": "_install_chunked_prefill._patched_remove", "full_name": "vllm_mlx.scheduler._install_chunked_prefill._patched_remove", "kind": "nested function", "signature": "def _patched_remove(uids_to_remove, _self=batch_gen)", "parameters": [ { "name": "uids_to_remove", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "_self", "kind": "positional or keyword", "annotation": "not annotated", "default": "batch_gen", "required": false, "description": "Optional positional or keyword input; defaults to `batch_gen`." } ], "return_annotation": "not annotated", "docstring": "Clear partial state if aborted request is being prefilled.", "summary": "Clear partial state if aborted request is being prefilled.", "implementation": "Nested Function `_install_chunked_prefill._patched_remove` calls `set`, `logger.info`, `mx.clear_cache`, `_orig_remove`.", "documented": true, "public": false, "addressable": false, "line": 680, "end_line": 691, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L680-L691", "decorators": [], "calls": [ "set", "logger.info", "mx.clear_cache", "_orig_remove" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_MTPStatsState", "qualname": "_MTPStatsState", "full_name": "vllm_mlx.scheduler._MTPStatsState", "kind": "class", "signature": "class _MTPStatsState", "parameters": [ { "name": "counters", "kind": "field", "annotation": "Dict[str, int]", "default": "field(default_factory=lambda: {'attempted': 0, 'accepted': 0, 'rejected': 0, 'errors': 0})", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=lambda: {'attempted': 0, 'accepted': 0, 'rejected': 0, 'errors': 0})`." }, { "name": "bypass_counts", "kind": "field", "annotation": "Dict[str, int]", "default": "field(default_factory=lambda: {'prefill': 0, 'no_active_batch': 0, 'cache_mismatch': 0})", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=lambda: {'prefill': 0, 'no_active_batch': 0, 'cache_mismatch': 0})`." }, { "name": "lock", "kind": "field", "annotation": "Any", "default": "field(default_factory=Lock)", "required": false, "description": "Optional constructor field; defaults to `field(default_factory=Lock)`." } ], "return_annotation": "_MTPStatsState", "docstring": "Cumulative native-MTP counters shared across generator instances.", "summary": "Cumulative native-MTP counters shared across generator instances.", "implementation": "Class `_MTPStatsState` declares 0 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 701, "end_line": 719, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L701-L719", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_configure_chunked_prefill", "qualname": "_configure_chunked_prefill", "full_name": "vllm_mlx.scheduler._configure_chunked_prefill", "kind": "function", "signature": "def _configure_chunked_prefill(scheduler: 'Scheduler', batch_gen: 'BatchGenerator', budget: int, prompt_cache_save) -> None", "parameters": [ { "name": "scheduler", "kind": "positional or keyword", "annotation": "'Scheduler'", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "batch_gen", "kind": "positional or keyword", "annotation": "'BatchGenerator'", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "budget", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_cache_save", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Enable the matching legacy or native mlx-lm chunked-prefill API.", "summary": "Enable the matching legacy or native mlx-lm chunked-prefill API.", "implementation": "Function `_configure_chunked_prefill` calls `hasattr`, `scheduler._make_mid_prefill_save_callback`, `logger.info`, `_install_chunked_prefill`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 722, "end_line": 777, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L722-L777", "decorators": [], "calls": [ "hasattr", "scheduler._make_mid_prefill_save_callback", "logger.info", "_install_chunked_prefill", "all", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_install_mtp", "qualname": "_install_mtp", "full_name": "vllm_mlx.scheduler._install_mtp", "kind": "function", "signature": "def _install_mtp(batch_gen: 'BatchGenerator', model: Any, num_draft_tokens: int=1, optimistic: bool=False, stats_state: Optional['_MTPStatsState']=None) -> None", "parameters": [ { "name": "batch_gen", "kind": "positional or keyword", "annotation": "'BatchGenerator'", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_draft_tokens", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Optional positional or keyword input; defaults to `1`." }, { "name": "optimistic", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." }, { "name": "stats_state", "kind": "positional or keyword", "annotation": "Optional['_MTPStatsState']", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Monkey-patch a BatchGenerator to use MTP (Multi-Token Prediction)\nwith always-advance strategy for hybrid MambaCache + KVCache.\n\nFlow per generation step:\n1. Use skip_state logits/hidden OR run model forward -> sample primary\n2. MTP head drafts one token after primary\n3. Verify [primary, draft] in one model call (always advances cache)\n4. Accept: skip_state from pos 1, defer draft for next step emission\n Reject: trim KVCache by 1, skip_state from pos 0 (no cold start)\n5. Draft is emitted in the NEXT generation step after primary", "summary": "Monkey-patch a BatchGenerator to use MTP (Multi-Token Prediction) with always-advance strategy for hybrid MambaCache + KVCache.", "implementation": "Function `_install_mtp` calls `make_sampler`, `_MTPStatsState`, `logger.warning`, `logger.info`.", "documented": true, "public": false, "addressable": true, "line": 780, "end_line": 1262, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L780-L1262", "decorators": [], "calls": [ "make_sampler", "_MTPStatsState", "logger.warning", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_get_mtp_stats", "qualname": "_install_mtp._get_mtp_stats", "full_name": "vllm_mlx.scheduler._install_mtp._get_mtp_stats", "kind": "nested function", "signature": "def _get_mtp_stats() -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "", "summary": "Nested Function `_install_mtp._get_mtp_stats` calls `dict`; returns `{'enabled': True, 'requested_draft_tokens': num_draft_tokens, 'effective_draft_tokens': 1, 'mode': 'always_advance_opti…`.", "implementation": "Nested Function `_install_mtp._get_mtp_stats` calls `dict`; returns `{'enabled': True, 'requested_draft_tokens': num_draft_tokens, 'effective_draft_tokens': 1, 'mode': 'always_advance_opti…`.", "documented": false, "public": false, "addressable": false, "line": 823, "end_line": 845, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L823-L845", "decorators": [], "calls": [ "dict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'enabled': True, 'requested_draft_tokens': num_draft_tokens, 'effective_draft_tokens': 1, 'mode': 'always_advance_opti…" ], "awaits": false, "yields": false }, { "name": "_mtp_bypass_reasons", "qualname": "_install_mtp._mtp_bypass_reasons", "full_name": "vllm_mlx.scheduler._install_mtp._mtp_bypass_reasons", "kind": "nested function", "signature": "def _mtp_bypass_reasons(input_tokens, prompt_cache)", "parameters": [ { "name": "input_tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_install_mtp._mtp_bypass_reasons` calls `reasons.append`; returns `reasons`.", "implementation": "Nested Function `_install_mtp._mtp_bypass_reasons` calls `reasons.append`; returns `reasons`.", "documented": false, "public": false, "addressable": false, "line": 849, "end_line": 857, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L849-L857", "decorators": [], "calls": [ "reasons.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "reasons" ], "awaits": false, "yields": false }, { "name": "_record_mtp_bypass", "qualname": "_install_mtp._record_mtp_bypass", "full_name": "vllm_mlx.scheduler._install_mtp._record_mtp_bypass", "kind": "nested function", "signature": "def _record_mtp_bypass(reasons) -> None", "parameters": [ { "name": "reasons", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Nested Function `_install_mtp._record_mtp_bypass` contains no state mutation, call, raise, return, await, or yield.", "implementation": "Nested Function `_install_mtp._record_mtp_bypass` contains no state mutation, call, raise, return, await, or yield.", "documented": false, "public": false, "addressable": false, "line": 859, "end_line": 862, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L859-L862", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_mtp_step", "qualname": "_install_mtp._mtp_step", "full_name": "vllm_mlx.scheduler._install_mtp._mtp_step", "kind": "nested function", "signature": "def _mtp_step(input_tokens, prompt_cache, samplers, logits_processors, tokens)", "parameters": [ { "name": "input_tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "samplers", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits_processors", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Extended _step with MTP always-advance strategy.\n\nEvery step (after skip):\n1. Use skip_state logits/hidden OR run model forward\n2. Sample primary token P\n3. MTP head drafts token D\n4. Verify [P, D] in one model call (always advances cache)\n5. Accept: skip_state from position 1 (after D), defer D\n Reject: trim KVCache by 1, skip_state from position 0 (after P)\n\nNo snapshot/restore — eliminates cold starts after rejection.\nMambaCache layers accept minor pollution on reject (exponential decay).\n\nDuring prefill (multi-token input), MTP is skipped entirely.", "summary": "Extended _step with MTP always-advance strategy.", "implementation": "Nested Function `_install_mtp._mtp_step` calls `_mtp_bypass_reasons`, `_record_mtp_bypass`, `_orig_step`, `model`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 864, "end_line": 1138, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L864-L1138", "decorators": [], "calls": [ "_mtp_bypass_reasons", "_record_mtp_bypass", "_orig_step", "model", "isinstance", "_normalize_logits_processors", "any", "logger.debug", "sum", "len", "range", "processor", "processed_logits.append", "mx.concatenate", "mx.logsumexp", "sample_sampler", "all_samples.append", "batch_gen.sampler", "list", "model.mtp_forward", "_draft_sampler", "enumerate", "hasattr", "_c.is_trimmable", "s.copy", "mx.async_eval", "mx.argmax", "mx.eval", "verify_pred.tolist", "draft_tokens.tolist", "c.is_trimmable", "c.trim", "_rnn_snapshots.items", "_deferred_drafts.pop" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_orig_step(input_tokens, prompt_cache, samplers, logits_processors, tokens)", "(primary_tokens, list(logprobs))" ], "awaits": false, "yields": false }, { "name": "_mtp_next", "qualname": "_install_mtp._mtp_next", "full_name": "vllm_mlx.scheduler._install_mtp._mtp_next", "kind": "nested function", "signature": "def _mtp_next(self=batch_gen)", "parameters": [], "return_annotation": "not annotated", "docstring": "Wrapper around _next that emits deferred MTP draft tokens.\n\nAfter each primary token, if the previous step's MTP draft was\naccepted, it is emitted as an additional response.", "summary": "Wrapper around _next that emits deferred MTP draft tokens.", "implementation": "Nested Function `_install_mtp._mtp_next` updates `self.active_batch`; calls `_deferred_drafts.clear`, `_deferred_drafts.pop`, `self._inner_next`, `set`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 1147, "end_line": 1247, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1147-L1247", "decorators": [], "calls": [ "_deferred_drafts.clear", "_deferred_drafts.pop", "self._inner_next", "set", "augmented.append", "prev_deferred.pop", "draft_info['token_array'].item", "self.Response", "draft_end_uids.add", "enumerate", "mx.concatenate", "mx.array", "batch.extract_cache", "self.active_batch.filter" ], "state_reads": [ "self.active_batch", "self.active_batch.uids", "self._inner_next", "self.stop_tokens", "self.Response", "self.active_batch.filter" ], "state_writes": [ "self.active_batch" ], "raises": [], "return_expressions": [ "responses", "augmented" ], "awaits": false, "yields": false }, { "name": "_mtp_status_snapshot", "qualname": "_mtp_status_snapshot", "full_name": "vllm_mlx.scheduler._mtp_status_snapshot", "kind": "function", "signature": "def _mtp_status_snapshot(batch_generator) -> Dict[str, Any]", "parameters": [ { "name": "batch_generator", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Dict[str, Any]", "docstring": "", "summary": "Function `_mtp_status_snapshot` calls `getattr`, `callable`, `get_mtp_stats`; has 2 explicit return paths.", "implementation": "Function `_mtp_status_snapshot` calls `getattr`, `callable`, `get_mtp_stats`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1265, "end_line": 1269, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1265-L1269", "decorators": [], "calls": [ "getattr", "callable", "get_mtp_stats" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'mtp': get_mtp_stats()}", "{}" ], "awaits": false, "yields": false }, { "name": "Scheduler", "qualname": "Scheduler", "full_name": "vllm_mlx.scheduler.Scheduler", "kind": "class", "signature": "class Scheduler", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model" }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The tokenizer" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[SchedulerConfig]", "default": "None", "required": false, "description": "Scheduler configuration" } ], "return_annotation": "Scheduler", "docstring": "Scheduler for continuous batching using mlx-lm BatchGenerator.\n\nThis scheduler manages the lifecycle of requests:\n1. Requests arrive and are added to the waiting queue\n2. Scheduler moves requests from waiting to running (via BatchGenerator)\n3. BatchGenerator processes all running requests together\n4. Finished requests are removed and outputs returned\n\nThe key insight is that mlx-lm's BatchGenerator already implements\ncontinuous batching at the token level, so we use it as the backend.", "summary": "Scheduler for continuous batching using mlx-lm BatchGenerator.", "implementation": "Class `Scheduler` declares 52 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 1272, "end_line": 3518, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1272-L3518", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "Scheduler.__init__", "full_name": "vllm_mlx.scheduler.Scheduler.__init__", "kind": "method", "signature": "def __init__(self, model: Any, tokenizer: Any, config: Optional[SchedulerConfig]=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The MLX model" }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The tokenizer" }, { "name": "config", "kind": "positional or keyword", "annotation": "Optional[SchedulerConfig]", "default": "None", "required": false, "description": "Scheduler configuration" } ], "return_annotation": "not annotated", "docstring": "Initialize the scheduler.\n\nArgs:\n model: The MLX model\n tokenizer: The tokenizer\n config: Scheduler configuration", "summary": "Initialize the scheduler.", "implementation": "Method `Scheduler.__init__` updates `self.model`, `self.tokenizer`, `self.config`, `self._actual_tokenizer`; calls `SchedulerConfig`, `self._get_actual_tokenizer`, `deque`, `set`.", "documented": true, "public": false, "addressable": true, "line": 1286, "end_line": 1402, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1286-L1402", "decorators": [], "calls": [ "SchedulerConfig", "self._get_actual_tokenizer", "deque", "set", "PagedCacheManager", "BlockAwarePrefixCache", "logger.info", "MemoryCacheConfig", "MemoryAwarePrefixCache", "SSDCacheConfig", "SSDCacheTier", "self._ssd_tier.start_writer", "self._ssd_tier.reconcile", "self.memory_aware_cache.set_ssd_tier", "PrefixCacheManager", "_MTPStatsState" ], "state_reads": [ "self._get_actual_tokenizer", "self.config.enable_prefix_cache", "self.config", "self.config.use_paged_cache", "self.config.paged_cache_block_size", "self.config.max_cache_blocks", "self.paged_cache_manager", "self.config.use_memory_aware_cache", "self.config.cache_memory_mb", "self.config.cache_memory_percent", "self.config.kv_cache_quantization", "self.config.kv_cache_quantization_bits", "self.config.kv_cache_quantization_group_size", "self.config.kv_cache_min_quantize_tokens", "self.memory_aware_cache.memory_limit_mb", "self.memory_aware_cache", "self.config.ssd_cache_dir", "self.config.ssd_cache_max_gb", "self._ssd_tier.start_writer", "self._ssd_tier", "self._ssd_tier.reconcile", "self.memory_aware_cache.set_ssd_tier", "self.config.prefix_cache_size" ], "state_writes": [ "self.model", "self.tokenizer", "self.config", "self._actual_tokenizer", "self._detokenizer_pool", "self.waiting", "self.running", "self.requests", "self.finished_req_ids", "self.request_id_to_uid", "self.uid_to_request_id", "self.batch_generator", "self._current_sampler_params", "self.prefix_cache", "self.memory_aware_cache", "self.paged_cache_manager", "self.block_aware_cache", "self._ssd_tier", "self._pending_abort_ids", "self.num_requests_processed", "self.total_prompt_tokens", "self.total_completion_tokens", "self._mtp_stats_state", "self._step_count", "self._clear_cache_interval", "self._memory_log_interval" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_get_actual_tokenizer", "qualname": "Scheduler._get_actual_tokenizer", "full_name": "vllm_mlx.scheduler.Scheduler._get_actual_tokenizer", "kind": "method", "signature": "def _get_actual_tokenizer(self, tokenizer: Any) -> Any", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Get the actual tokenizer from a processor or tokenizer.\n\nMLLM models use processors (e.g., Qwen3VLProcessor) which wrap\nthe tokenizer. This method extracts the actual tokenizer.", "summary": "Get the actual tokenizer from a processor or tokenizer.", "implementation": "Method `Scheduler._get_actual_tokenizer` calls `hasattr`, `callable`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1404, "end_line": 1418, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1404-L1418", "decorators": [], "calls": [ "hasattr", "callable" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tokenizer", "tokenizer.tokenizer" ], "awaits": false, "yields": false }, { "name": "_decode_tokens", "qualname": "Scheduler._decode_tokens", "full_name": "vllm_mlx.scheduler.Scheduler._decode_tokens", "kind": "method", "signature": "def _decode_tokens(self, token_ids: List[int]) -> str", "parameters": [ { "name": "token_ids", "kind": "positional or keyword", "annotation": "List[int]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Decode token IDs to text, handling both tokenizers and processors.", "summary": "Decode token IDs to text, handling both tokenizers and processors.", "implementation": "Method `Scheduler._decode_tokens` calls `self._actual_tokenizer.decode`; returns `self._actual_tokenizer.decode(token_ids)`.", "documented": true, "public": false, "addressable": true, "line": 1420, "end_line": 1424, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1420-L1424", "decorators": [], "calls": [ "self._actual_tokenizer.decode" ], "state_reads": [ "self._actual_tokenizer.decode", "self._actual_tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "self._actual_tokenizer.decode(token_ids)" ], "awaits": false, "yields": false }, { "name": "_get_detokenizer", "qualname": "Scheduler._get_detokenizer", "full_name": "vllm_mlx.scheduler.Scheduler._get_detokenizer", "kind": "method", "signature": "def _get_detokenizer(self, request_id: str) -> Any", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Get or create a streaming detokenizer for a request.", "summary": "Get or create a streaming detokenizer for a request.", "implementation": "Method `Scheduler._get_detokenizer` calls `NaiveStreamingDetokenizer`; returns `self._detokenizer_pool[request_id]`.", "documented": true, "public": false, "addressable": true, "line": 1426, "end_line": 1431, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1426-L1431", "decorators": [], "calls": [ "NaiveStreamingDetokenizer" ], "state_reads": [ "self._detokenizer_pool", "self._actual_tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "self._detokenizer_pool[request_id]" ], "awaits": false, "yields": false }, { "name": "_cleanup_detokenizer", "qualname": "Scheduler._cleanup_detokenizer", "full_name": "vllm_mlx.scheduler.Scheduler._cleanup_detokenizer", "kind": "method", "signature": "def _cleanup_detokenizer(self, request_id: str) -> None", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Remove the streaming detokenizer for a finished request.", "summary": "Remove the streaming detokenizer for a finished request.", "implementation": "Method `Scheduler._cleanup_detokenizer` calls `self._detokenizer_pool.pop`.", "documented": true, "public": false, "addressable": true, "line": 1433, "end_line": 1435, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1433-L1435", "decorators": [], "calls": [ "self._detokenizer_pool.pop" ], "state_reads": [ "self._detokenizer_pool.pop", "self._detokenizer_pool" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_get_stop_tokens", "qualname": "Scheduler._get_stop_tokens", "full_name": "vllm_mlx.scheduler.Scheduler._get_stop_tokens", "kind": "method", "signature": "def _get_stop_tokens(self) -> Set[int]", "parameters": [], "return_annotation": "Set[int]", "docstring": "Get stop token IDs from tokenizer or processor.", "summary": "Get stop token IDs from tokenizer or processor.", "implementation": "Method `Scheduler._get_stop_tokens` calls `set`, `hasattr`, `isinstance`, `stop_tokens.update`; returns `stop_tokens`.", "documented": true, "public": false, "addressable": true, "line": 1437, "end_line": 1455, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1437-L1455", "decorators": [], "calls": [ "set", "hasattr", "isinstance", "stop_tokens.update", "stop_tokens.add" ], "state_reads": [ "self.tokenizer", "self._actual_tokenizer" ], "state_writes": [], "raises": [], "return_expressions": [ "stop_tokens" ], "awaits": false, "yields": false }, { "name": "_create_batch_generator", "qualname": "Scheduler._create_batch_generator", "full_name": "vllm_mlx.scheduler.Scheduler._create_batch_generator", "kind": "method", "signature": "def _create_batch_generator(self, sampling_params: SamplingParams) -> BatchGenerator", "parameters": [ { "name": "sampling_params", "kind": "positional or keyword", "annotation": "SamplingParams", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BatchGenerator", "docstring": "Create a BatchGenerator with the given sampling parameters.", "summary": "Create a BatchGenerator with the given sampling parameters.", "implementation": "Method `Scheduler._create_batch_generator` calls `make_sampler`, `self._get_stop_tokens`, `stop_tokens.update`, `BatchGenerator`; returns `bg`.", "documented": true, "public": false, "addressable": true, "line": 1457, "end_line": 1539, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1457-L1539", "decorators": [], "calls": [ "make_sampler", "self._get_stop_tokens", "stop_tokens.update", "BatchGenerator", "self._make_prompt_cache_save_callback", "_configure_chunked_prefill", "hasattr", "_install_prompt_cache_save", "_install_mtp", "logger.warning" ], "state_reads": [ "self._get_stop_tokens", "self.model", "self.config.prefill_batch_size", "self.config", "self.config.completion_batch_size", "self.config.prefill_step_size", "self.config.chunked_prefill_tokens", "self.memory_aware_cache", "self._make_prompt_cache_save_callback", "self.config.enable_mtp", "self.model.mtp", "self.config.mtp_num_draft_tokens", "self.config.mtp_optimistic", "self._mtp_stats_state" ], "state_writes": [], "raises": [], "return_expressions": [ "bg" ], "awaits": false, "yields": false }, { "name": "_prefill_progress", "qualname": "Scheduler._create_batch_generator._prefill_progress", "full_name": "vllm_mlx.scheduler.Scheduler._create_batch_generator._prefill_progress", "kind": "nested function", "signature": "def _prefill_progress(progress_list)", "parameters": [ { "name": "progress_list", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Log prefill progress for each uid chunk.", "summary": "Log prefill progress for each uid chunk.", "implementation": "Nested Function `Scheduler._create_batch_generator._prefill_progress` calls `self.uid_to_request_id.get`, `logger.info`, `isinstance`.", "documented": true, "public": false, "addressable": false, "line": 1472, "end_line": 1479, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1472-L1479", "decorators": [], "calls": [ "self.uid_to_request_id.get", "logger.info", "isinstance" ], "state_reads": [ "self.uid_to_request_id.get", "self.uid_to_request_id" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_make_prompt_cache_save_callback", "qualname": "Scheduler._make_prompt_cache_save_callback", "full_name": "vllm_mlx.scheduler.Scheduler._make_prompt_cache_save_callback", "kind": "method", "signature": "def _make_prompt_cache_save_callback(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Create a callback that stores prompt-only KV/Mamba cache.\n\nCalled from ``_generation_step`` right before the first output token\nis fed into the model. At that point ``num_tokens == 0`` and the\nbatch cache contains the exact prompt-only state (correct for both\nKVCache and MambaCache/ArraysCache layers).\n\nThe cache is stored with key = prompt_token_ids so that a future\nrequest with the identical prompt gets an exact hit.", "summary": "Create a callback that stores prompt-only KV/Mamba cache.", "implementation": "Method `Scheduler._make_prompt_cache_save_callback` returns `_prompt_cache_save`.", "documented": true, "public": false, "addressable": true, "line": 1541, "end_line": 1585, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1541-L1585", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_prompt_cache_save" ], "awaits": false, "yields": false }, { "name": "_prompt_cache_save", "qualname": "Scheduler._make_prompt_cache_save_callback._prompt_cache_save", "full_name": "vllm_mlx.scheduler.Scheduler._make_prompt_cache_save_callback._prompt_cache_save", "kind": "nested function", "signature": "def _prompt_cache_save(uid, extracted_cache)", "parameters": [ { "name": "uid", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "extracted_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `Scheduler._make_prompt_cache_save_callback._prompt_cache_save` calls `self.uid_to_request_id.get`, `self.requests.get`, `list`, `_trim_cache_offset`; returns `None`.", "implementation": "Nested Function `Scheduler._make_prompt_cache_save_callback._prompt_cache_save` calls `self.uid_to_request_id.get`, `self.requests.get`, `list`, `_trim_cache_offset`; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 1554, "end_line": 1583, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1554-L1583", "decorators": [], "calls": [ "self.uid_to_request_id.get", "self.requests.get", "list", "_trim_cache_offset", "_time.monotonic", "self.memory_aware_cache.store", "logger.info", "len" ], "state_reads": [ "self.uid_to_request_id.get", "self.uid_to_request_id", "self.requests.get", "self.requests", "self.memory_aware_cache.store", "self.memory_aware_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_make_mid_prefill_save_callback", "qualname": "Scheduler._make_mid_prefill_save_callback", "full_name": "vllm_mlx.scheduler.Scheduler._make_mid_prefill_save_callback", "kind": "method", "signature": "def _make_mid_prefill_save_callback(self, save_interval: int)", "parameters": [ { "name": "save_interval", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Create a callback for saving intermediate KV cache during chunked prefill.\n\nThe callback is called after each chunk with (uid, processed_tokens,\nprompt_cache). It extracts the cache state (immutable MLX array\nsnapshots), reconstructs KVCache objects, and stores them in the\nmemory-aware prefix cache so that a subsequent request with the same\nprompt prefix can skip the already-computed tokens.", "summary": "Create a callback for saving intermediate KV cache during chunked prefill.", "implementation": "Method `Scheduler._make_mid_prefill_save_callback` returns `_mid_prefill_save`.", "documented": true, "public": false, "addressable": true, "line": 1587, "end_line": 1655, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1587-L1655", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_mid_prefill_save" ], "awaits": false, "yields": false }, { "name": "_mid_prefill_save", "qualname": "Scheduler._make_mid_prefill_save_callback._mid_prefill_save", "full_name": "vllm_mlx.scheduler.Scheduler._make_mid_prefill_save_callback._mid_prefill_save", "kind": "nested function", "signature": "def _mid_prefill_save(uid, processed_tokens, prompt_cache)", "parameters": [ { "name": "uid", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "processed_tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `Scheduler._make_mid_prefill_save_callback._mid_prefill_save` calls `self.uid_to_request_id.get`, `self.requests.get`, `getattr`, `self._extract_cache_states`; returns `None`.", "implementation": "Nested Function `Scheduler._make_mid_prefill_save_callback._mid_prefill_save` calls `self.uid_to_request_id.get`, `self.requests.get`, `getattr`, `self._extract_cache_states`; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 1598, "end_line": 1653, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1598-L1653", "decorators": [], "calls": [ "self.uid_to_request_id.get", "self.requests.get", "getattr", "self._extract_cache_states", "self._reconstruct_cache_from_states", "list", "self.memory_aware_cache.remove", "_time.monotonic", "self.memory_aware_cache.store", "tuple", "logger.info", "len", "logger.debug" ], "state_reads": [ "self.uid_to_request_id.get", "self.uid_to_request_id", "self.requests.get", "self.requests", "self._extract_cache_states", "self._reconstruct_cache_from_states", "self.memory_aware_cache.remove", "self.memory_aware_cache", "self.memory_aware_cache.store" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_close_batch_generator", "qualname": "Scheduler._close_batch_generator", "full_name": "vllm_mlx.scheduler.Scheduler._close_batch_generator", "kind": "method", "signature": "def _close_batch_generator(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Properly close BatchGenerator to restore wired_limit.", "summary": "Properly close BatchGenerator to restore wired_limit.", "implementation": "Method `Scheduler._close_batch_generator` updates `self.batch_generator`; calls `hasattr`, `self.batch_generator.close`, `logger.debug`.", "documented": true, "public": false, "addressable": true, "line": 1657, "end_line": 1665, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1657-L1665", "decorators": [], "calls": [ "hasattr", "self.batch_generator.close", "logger.debug" ], "state_reads": [ "self.batch_generator", "self.batch_generator.close" ], "state_writes": [ "self.batch_generator" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_ensure_batch_generator", "qualname": "Scheduler._ensure_batch_generator", "full_name": "vllm_mlx.scheduler.Scheduler._ensure_batch_generator", "kind": "method", "signature": "def _ensure_batch_generator(self, sampling_params: SamplingParams) -> None", "parameters": [ { "name": "sampling_params", "kind": "positional or keyword", "annotation": "SamplingParams", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Ensure BatchGenerator exists with compatible settings.", "summary": "Ensure BatchGenerator exists with compatible settings.", "implementation": "Method `Scheduler._ensure_batch_generator` updates `self.batch_generator`, `self._current_sampler_params`; calls `logger.warning`, `len`, `hasattr`, `logger.info`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1667, "end_line": 1709, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1667-L1709", "decorators": [], "calls": [ "logger.warning", "len", "hasattr", "logger.info", "self._close_batch_generator", "self._create_batch_generator" ], "state_reads": [ "self.batch_generator", "self._current_sampler_params", "self.running", "self.memory_aware_cache", "self.memory_aware_cache._entries", "self.prefix_cache", "self._close_batch_generator", "self._create_batch_generator" ], "state_writes": [ "self.batch_generator", "self._current_sampler_params" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_validate_cache", "qualname": "Scheduler._validate_cache", "full_name": "vllm_mlx.scheduler.Scheduler._validate_cache", "kind": "method", "signature": "def _validate_cache(self, cache: Any) -> bool", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The cache object to validate" } ], "return_annotation": "bool", "docstring": "Validate that a cache object is usable.\n\nChecks for None references AND shape compatibility. Restored\ncache entries must have batch_size == 1 (single sequence) so\nthey can be merged into the running batch by _merge_caches.\nA shape mismatch here (e.g. batch=2 from a stale entry) would\ncause a concatenation crash inside _merge_caches.\n\nArgs:\n cache: The cache object to validate\n\nReturns:\n True if cache is valid and usable", "summary": "Validate that a cache object is usable.", "implementation": "Method `Scheduler._validate_cache` calls `isinstance`, `len`, `hasattr`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1711, "end_line": 1769, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1711-L1769", "decorators": [], "calls": [ "isinstance", "len", "hasattr", "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_extract_cache_states", "qualname": "Scheduler._extract_cache_states", "full_name": "vllm_mlx.scheduler.Scheduler._extract_cache_states", "kind": "method", "signature": "def _extract_cache_states(self, raw_cache: List[Any]) -> List[Dict[str, Any]]", "parameters": [ { "name": "raw_cache", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "List of KVCache objects from mlx-lm" } ], "return_annotation": "List[Dict[str, Any]]", "docstring": "Extract actual tensor state from each layer cache.\n\nThis extracts the real KV data using mlx-lm's cache.state property,\nallowing the data to be stored and reconstructed later even after\nthe BatchGenerator is recreated.\n\nArgs:\n raw_cache: List of KVCache objects from mlx-lm\n\nReturns:\n List of dicts with {state: (keys, values), meta_state: (offset,), class_name: str}", "summary": "Extract actual tensor state from each layer cache.", "implementation": "Method `Scheduler._extract_cache_states` calls `hasattr`, `extracted.append`, `type`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1771, "end_line": 1806, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1771-L1806", "decorators": [], "calls": [ "hasattr", "extracted.append", "type", "logger.debug", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "extracted if len(extracted) == len(raw_cache) else []" ], "awaits": false, "yields": false }, { "name": "_reconstruct_cache_from_states", "qualname": "Scheduler._reconstruct_cache_from_states", "full_name": "vllm_mlx.scheduler.Scheduler._reconstruct_cache_from_states", "kind": "method", "signature": "def _reconstruct_cache_from_states(self, extracted_states: List[Dict[str, Any]]) -> Optional[List[Any]]", "parameters": [ { "name": "extracted_states", "kind": "positional or keyword", "annotation": "List[Dict[str, Any]]", "default": "", "required": true, "description": "List of dicts from _extract_cache_states()" } ], "return_annotation": "Optional[List[Any]]", "docstring": "Reconstruct cache objects from extracted cache states.\n\nThis is the inverse of _extract_cache_states(). Uses mlx-lm's\n_BaseCache.from_state() to reconstruct any cache type (KVCache,\nMambaCache, etc.) from its state/meta_state.\n\nArgs:\n extracted_states: List of dicts from _extract_cache_states()\n\nReturns:\n List of cache objects, or None if reconstruction fails", "summary": "Reconstruct cache objects from extracted cache states.", "implementation": "Method `Scheduler._reconstruct_cache_from_states` calls `layer_state.get`, `hasattr`, `_KVCache`, `cache_cls.from_state`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1808, "end_line": 1872, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1808-L1872", "decorators": [], "calls": [ "layer_state.get", "hasattr", "_KVCache", "cache_cls.from_state", "len", "KVCache", "int", "caches.append", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "caches" ], "awaits": false, "yields": false }, { "name": "add_request", "qualname": "Scheduler.add_request", "full_name": "vllm_mlx.scheduler.Scheduler.add_request", "kind": "method", "signature": "def add_request(self, request: Request) -> None", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "The request to add" } ], "return_annotation": "None", "docstring": "Add a new request to the scheduler.\n\nArgs:\n request: The request to add", "summary": "Add a new request to the scheduler.", "implementation": "Method `Scheduler.add_request` calls `ValueError`, `isinstance`, `hasattr`, `self.tokenizer.encode`; can raise `ValueError`, `AttributeError`.", "documented": true, "public": true, "addressable": true, "line": 1874, "end_line": 1997, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1874-L1997", "decorators": [], "calls": [ "ValueError", "isinstance", "hasattr", "self.tokenizer.encode", "self.tokenizer.tokenizer.encode", "AttributeError", "type", "list", "len", "self.block_aware_cache.fetch_cache", "self.block_aware_cache.reconstruct_cache", "logger.debug", "_time.monotonic", "self.memory_aware_cache.fetch", "logger.info", "self.memory_aware_cache.check_ssd", "self.prefix_cache.fetch_cache", "self.waiting.append" ], "state_reads": [ "self.requests", "self.tokenizer", "self.tokenizer.encode", "self.tokenizer.tokenizer", "self.tokenizer.tokenizer.encode", "self.block_aware_cache", "self.block_aware_cache.fetch_cache", "self.block_aware_cache.reconstruct_cache", "self.memory_aware_cache", "self.memory_aware_cache.fetch", "self.memory_aware_cache._last_match_type", "self.memory_aware_cache._entries", "self._ssd_tier", "self.memory_aware_cache.check_ssd", "self.prefix_cache", "self.prefix_cache.fetch_cache", "self.waiting.append", "self.waiting" ], "state_writes": [], "raises": [ "ValueError", "AttributeError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "abort_request", "qualname": "Scheduler.abort_request", "full_name": "vllm_mlx.scheduler.Scheduler.abort_request", "kind": "method", "signature": "def abort_request(self, request_id: str) -> bool", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The request ID to abort" } ], "return_annotation": "bool", "docstring": "Queue request for abort. Thread-safe, called from any thread.\n\nThe actual abort is deferred to the executor thread (inside step())\nto avoid race conditions with in-flight Metal GPU operations.\n\nArgs:\n request_id: The request ID to abort\n\nReturns:\n True (abort is always enqueued)", "summary": "Queue request for abort.", "implementation": "Method `Scheduler.abort_request` calls `self._pending_abort_ids.add`, `logger.info`; returns `True`.", "documented": true, "public": true, "addressable": true, "line": 1999, "end_line": 2014, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L1999-L2014", "decorators": [], "calls": [ "self._pending_abort_ids.add", "logger.info" ], "state_reads": [ "self._pending_abort_ids.add", "self._pending_abort_ids" ], "state_writes": [], "raises": [], "return_expressions": [ "True" ], "awaits": false, "yields": false }, { "name": "_process_pending_aborts", "qualname": "Scheduler._process_pending_aborts", "full_name": "vllm_mlx.scheduler.Scheduler._process_pending_aborts", "kind": "method", "signature": "def _process_pending_aborts(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Drain and process pending abort requests. Called from executor thread.", "summary": "Drain and process pending abort requests.", "implementation": "Method `Scheduler._process_pending_aborts` calls `self._pending_abort_ids.pop`, `self._do_abort_request`.", "documented": true, "public": false, "addressable": true, "line": 2016, "end_line": 2020, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2016-L2020", "decorators": [], "calls": [ "self._pending_abort_ids.pop", "self._do_abort_request" ], "state_reads": [ "self._pending_abort_ids", "self._pending_abort_ids.pop", "self._do_abort_request" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_do_abort_request", "qualname": "Scheduler._do_abort_request", "full_name": "vllm_mlx.scheduler.Scheduler._do_abort_request", "kind": "method", "signature": "def _do_abort_request(self, request_id: str) -> bool", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The request ID to abort" } ], "return_annotation": "bool", "docstring": "Actually abort a request. Must be called from the executor thread.\n\nHandles the case where the request was already removed from\nself.requests by _cleanup_request() but still lives in the\nBatchGenerator (e.g. in _partial or active_batch).\n\nArgs:\n request_id: The request ID to abort\n\nReturns:\n True if any cleanup was performed, False otherwise", "summary": "Actually abort a request.", "implementation": "Method `Scheduler._do_abort_request` updates `self.total_completion_tokens`, `self.total_prompt_tokens`; calls `self.requests.get`, `self.waiting.remove`, `self.batch_generator.remove`, `request.set_finished`; returns `True`.", "documented": true, "public": false, "addressable": true, "line": 2022, "end_line": 2087, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2022-L2087", "decorators": [], "calls": [ "self.requests.get", "self.waiting.remove", "self.batch_generator.remove", "request.set_finished", "self.finished_req_ids.add", "self._cleanup_detokenizer", "mx.clear_cache", "logger.info", "len" ], "state_reads": [ "self.requests.get", "self.requests", "self.waiting.remove", "self.waiting", "self.request_id_to_uid", "self.batch_generator", "self.batch_generator.remove", "self.uid_to_request_id", "self.running", "self.finished_req_ids.add", "self.finished_req_ids", "self._cleanup_detokenizer" ], "state_writes": [ "self.total_completion_tokens", "self.total_prompt_tokens" ], "raises": [], "return_expressions": [ "True" ], "awaits": false, "yields": false }, { "name": "has_requests", "qualname": "Scheduler.has_requests", "full_name": "vllm_mlx.scheduler.Scheduler.has_requests", "kind": "method", "signature": "def has_requests(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if there are any pending or running requests.", "summary": "Check if there are any pending or running requests.", "implementation": "Method `Scheduler.has_requests` calls `bool`; returns `bool(self.waiting or self.running)`.", "documented": true, "public": true, "addressable": true, "line": 2089, "end_line": 2091, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2089-L2091", "decorators": [], "calls": [ "bool" ], "state_reads": [ "self.waiting", "self.running" ], "state_writes": [], "raises": [], "return_expressions": [ "bool(self.waiting or self.running)" ], "awaits": false, "yields": false }, { "name": "get_num_waiting", "qualname": "Scheduler.get_num_waiting", "full_name": "vllm_mlx.scheduler.Scheduler.get_num_waiting", "kind": "method", "signature": "def get_num_waiting(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get number of waiting requests.", "summary": "Get number of waiting requests.", "implementation": "Method `Scheduler.get_num_waiting` calls `len`; returns `len(self.waiting)`.", "documented": true, "public": true, "addressable": true, "line": 2093, "end_line": 2095, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2093-L2095", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.waiting" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.waiting)" ], "awaits": false, "yields": false }, { "name": "get_num_running", "qualname": "Scheduler.get_num_running", "full_name": "vllm_mlx.scheduler.Scheduler.get_num_running", "kind": "method", "signature": "def get_num_running(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get number of running requests.", "summary": "Get number of running requests.", "implementation": "Method `Scheduler.get_num_running` calls `len`; returns `len(self.running)`.", "documented": true, "public": true, "addressable": true, "line": 2097, "end_line": 2099, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2097-L2099", "decorators": [], "calls": [ "len" ], "state_reads": [ "self.running" ], "state_writes": [], "raises": [], "return_expressions": [ "len(self.running)" ], "awaits": false, "yields": false }, { "name": "_schedule_waiting", "qualname": "Scheduler._schedule_waiting", "full_name": "vllm_mlx.scheduler.Scheduler._schedule_waiting", "kind": "method", "signature": "def _schedule_waiting(self) -> List[Request]", "parameters": [], "return_annotation": "List[Request]", "docstring": "Move requests from waiting queue to running.\n\nReturns:\n List of requests that were scheduled", "summary": "Move requests from waiting queue to running.", "implementation": "Method `Scheduler._schedule_waiting` updates `self.total_prompt_tokens`; calls `self._try_promote_ssd_pending`, `len`, `self.waiting.popleft`, `self._ensure_batch_generator`; returns `scheduled`.", "documented": true, "public": false, "addressable": true, "line": 2101, "end_line": 2276, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2101-L2276", "decorators": [], "calls": [ "self._try_promote_ssd_pending", "len", "self.waiting.popleft", "self._ensure_batch_generator", "self.waiting.appendleft", "getattr", "logger.debug", "make_prompt_cache", "self._validate_cache", "combined_lp.extend", "make_logits_processors", "logger.info", "self.batch_generator.insert", "logger.warning", "scheduled.append" ], "state_reads": [ "self._ssd_tier", "self._try_promote_ssd_pending", "self.waiting", "self.running", "self.config.max_num_seqs", "self.config", "self.waiting.popleft", "self._ensure_batch_generator", "self.batch_generator", "self.waiting.appendleft", "self.config.max_kv_size", "self.model", "self._validate_cache", "self.batch_generator.insert", "self.request_id_to_uid", "self.uid_to_request_id" ], "state_writes": [ "self.total_prompt_tokens" ], "raises": [], "return_expressions": [ "scheduled" ], "awaits": false, "yields": false }, { "name": "_copy_cache_state", "qualname": "Scheduler._copy_cache_state", "full_name": "vllm_mlx.scheduler.Scheduler._copy_cache_state", "kind": "method", "signature": "def _copy_cache_state(value: Any) -> Any", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Deep-copy a cache ``state`` payload.\n\nSharing the arrays is not safe: RotatingKVCache writes into its ring\nbuffer and PoolingCache writes into its remainder buffer, both in\nplace, so a snapshot that aliases them would be rewritten by the very\ngeneration it is supposed to predate. ``x + 0`` forces a fresh array\nwhile staying on the GPU.", "summary": "Deep-copy a cache ``state`` payload.", "implementation": "Method `Scheduler._copy_cache_state` calls `isinstance`, `Scheduler._copy_cache_state`, `type(value)`, `type`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2279, "end_line": 2295, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2279-L2295", "decorators": [ "staticmethod" ], "calls": [ "isinstance", "Scheduler._copy_cache_state", "type(value)", "type" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "value + 0", "type(value)(copied) if isinstance(value, tuple) else copied", "value" ], "awaits": false, "yields": false }, { "name": "_prompt_output_entry_is_useless", "qualname": "Scheduler._prompt_output_entry_is_useless", "full_name": "vllm_mlx.scheduler.Scheduler._prompt_output_entry_is_useless", "kind": "method", "signature": "def _prompt_output_entry_is_useless(cache: Any) -> bool", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Would a prompt+output entry built from this cache ever be reusable?\n\nOnly via a trim: any later query is shorter than a prompt+output key, so\nthe generated tail has to come off first. When the cache cannot be\ntrimmed the entry is dead weight — and far from free, since each one\nholds a full-length KV copy and Metal runs out of buffers long before\nthe byte budget is reached.", "summary": "Would a prompt+output entry built from this cache ever be reusable?", "implementation": "Method `Scheduler._prompt_output_entry_is_useless` calls `can_trim_prompt_cache`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2303, "end_line": 2317, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2303-L2317", "decorators": [ "staticmethod" ], "calls": [ "can_trim_prompt_cache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "not can_trim_prompt_cache(cache)", "False" ], "awaits": false, "yields": false }, { "name": "_extract_cache_for_uid", "qualname": "Scheduler._extract_cache_for_uid", "full_name": "vllm_mlx.scheduler.Scheduler._extract_cache_for_uid", "kind": "method", "signature": "def _extract_cache_for_uid(self, uid: int) -> Any", "parameters": [ { "name": "uid", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Pull one sequence's cache out of the live BatchGenerator batch.", "summary": "Pull one sequence's cache out of the live BatchGenerator batch.", "implementation": "Method `Scheduler._extract_cache_for_uid` calls `getattr`, `extract`, `uids.index`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2319, "end_line": 2336, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2319-L2336", "decorators": [], "calls": [ "getattr", "extract", "uids.index", "logger.debug" ], "state_reads": [ "self.batch_generator" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "extract(uids.index(uid))" ], "awaits": false, "yields": false }, { "name": "_make_snapshot_destination", "qualname": "Scheduler._make_snapshot_destination", "full_name": "vllm_mlx.scheduler.Scheduler._make_snapshot_destination", "kind": "method", "signature": "def _make_snapshot_destination(self, live_cache: Any) -> Any", "parameters": [ { "name": "live_cache", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Build a destination cache with the same topology as the live one.\n\n``make_prompt_cache(model)`` is not a safe source for this. A plain\n``KVCache`` destination cannot take a ``RotatingKVCache``'s state or\nmeta_state; the assignment raises, the broad handler below logs a\nwarning, and the snapshot is silently never stored — on exactly the\nsliding-window configurations this feature exists for.\n\nDeriving it from ``config.max_kv_size`` instead is also wrong, which I\nonly found by measuring: ``_create_batch_generator`` does not pass\n``max_kv_size`` to ``BatchGenerator``, so with ``max_kv_size=512``\nconfigured the live layers were still plain ``KVCache`` and a\nconfig-derived destination mismatched in the opposite direction.\n\nSo mirror the live objects themselves. A shallow copy keeps the class\nand every scalar attribute (``max_size``, ``keep``, ``step``, ``_idx``)\nand the caller overwrites the arrays, which is the only part that must\nnot be shared.", "summary": "Build a destination cache with the same topology as the live one.", "implementation": "Method `Scheduler._make_snapshot_destination` calls `_mirror`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2338, "end_line": 2380, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2338-L2380", "decorators": [], "calls": [ "_mirror", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[_mirror(layer) for layer in live_cache]", "None" ], "awaits": false, "yields": false }, { "name": "_mirror", "qualname": "Scheduler._make_snapshot_destination._mirror", "full_name": "vllm_mlx.scheduler.Scheduler._make_snapshot_destination._mirror", "kind": "nested function", "signature": "def _mirror(layer: Any) -> Any", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "", "summary": "Nested Function `Scheduler._make_snapshot_destination._mirror` calls `getattr`, `_mirror`, `copy.copy`, `type(children)`; has 2 explicit return paths.", "implementation": "Nested Function `Scheduler._make_snapshot_destination._mirror` calls `getattr`, `_mirror`, `copy.copy`, `type(children)`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 2360, "end_line": 2370, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2360-L2370", "decorators": [], "calls": [ "getattr", "_mirror", "copy.copy", "type(children)", "type" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "container", "copy.copy(layer)" ], "awaits": false, "yields": false }, { "name": "_cache_coverage", "qualname": "Scheduler._cache_coverage", "full_name": "vllm_mlx.scheduler.Scheduler._cache_coverage", "kind": "method", "signature": "def _cache_coverage(cache: Any) -> int | None", "parameters": [ { "name": "cache", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int | None", "docstring": "How many tokens the live cache actually holds.\n\nContainers have to be descended into: ``CacheList`` carries no\n``offset`` of its own, so reading the attribute off the layer returns\nNone and the caller silently falls back to a prompt-only key — the\nmisalignment this is here to prevent, on exactly the architectures\n(DeepSeek-V4) that group several caches per layer.", "summary": "How many tokens the live cache actually holds.", "implementation": "Method `Scheduler._cache_coverage` calls `_offset_of`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2383, "end_line": 2409, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2383-L2409", "decorators": [ "staticmethod" ], "calls": [ "_offset_of" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "found", "None" ], "awaits": false, "yields": false }, { "name": "_offset_of", "qualname": "Scheduler._cache_coverage._offset_of", "full_name": "vllm_mlx.scheduler.Scheduler._cache_coverage._offset_of", "kind": "nested function", "signature": "def _offset_of(layer: Any) -> int | None", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int | None", "docstring": "", "summary": "Nested Function `Scheduler._cache_coverage._offset_of` calls `getattr`, `isinstance`, `_offset_of`; has 3 explicit return paths.", "implementation": "Nested Function `Scheduler._cache_coverage._offset_of` calls `getattr`, `isinstance`, `_offset_of`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 2393, "end_line": 2403, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2393-L2403", "decorators": [], "calls": [ "getattr", "isinstance", "_offset_of" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "offset", "found", "None" ], "awaits": false, "yields": false }, { "name": "_cache_key_for_snapshot", "qualname": "Scheduler._cache_key_for_snapshot", "full_name": "vllm_mlx.scheduler.Scheduler._cache_key_for_snapshot", "kind": "method", "signature": "def _cache_key_for_snapshot(self, request: Any, response: Any, raw_cache: Any) -> list[int] | None", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "response", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "raw_cache", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[int] | None", "docstring": "Key the entry by the tokens the cache covers, not by the prompt.\n\nThe snapshot is taken while processing the response that carries the\nfirst generated token, and by then the batch has already fed that token\nthrough the cache: measured ``prompt_len=5, cache_offset=6``. Storing\nthat under ``prompt_token_ids`` leaves every warm reuse one token ahead\nof its key.\n\nTrimming the overshoot off is not available here — these are precisely\nthe caches that cannot be trimmed — so the key is extended instead. The\nextra token is the first token of the reply, which the next turn's\nprompt also contains, so the entry still matches by strict prefix.\n\nReturns None rather than storing a misaligned entry.", "summary": "Key the entry by the tokens the cache covers, not by the prompt.", "implementation": "Method `Scheduler._cache_key_for_snapshot` calls `self._cache_coverage`, `list`, `logger.debug`, `', '.join`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2411, "end_line": 2468, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2411-L2468", "decorators": [], "calls": [ "self._cache_coverage", "list", "logger.debug", "', '.join", "sorted", "type", "len", "getattr", "int" ], "state_reads": [ "self._cache_coverage" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "prompt_ids", "prompt_ids + generated[:overshoot]" ], "awaits": false, "yields": false }, { "name": "_store_prompt_only_cache", "qualname": "Scheduler._store_prompt_only_cache", "full_name": "vllm_mlx.scheduler.Scheduler._store_prompt_only_cache", "kind": "method", "signature": "def _store_prompt_only_cache(self, request: Any, response: Any) -> None", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "response", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Store the post-prefill cache under the prompt tokens alone.\n\nCalled once per request, at the point where the cache covers exactly\nthe prompt. Entries keyed this way are reusable without any trimming,\nwhich is what models with sliding-window or pooled KV need.", "summary": "Store the post-prefill cache under the prompt tokens alone.", "implementation": "Method `Scheduler._store_prompt_only_cache` calls `getattr`, `len`, `callable`, `raw_cache`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 2470, "end_line": 2581, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2470-L2581", "decorators": [], "calls": [ "getattr", "len", "callable", "raw_cache", "self._extract_cache_for_uid", "self._prompt_output_entry_is_useless", "self._cache_key_for_snapshot", "_t.monotonic", "self._make_snapshot_destination", "zip", "self._copy_cache_state", "states.append", "mx.eval", "logger.debug", "self.memory_aware_cache.store", "logger.info", "logger.warning" ], "state_reads": [ "self.memory_aware_cache", "self.SNAPSHOT_REFRESH_TOKENS", "self._extract_cache_for_uid", "self._prompt_output_entry_is_useless", "self._cache_key_for_snapshot", "self._make_snapshot_destination", "self._copy_cache_state", "self.memory_aware_cache.store", "self.memory_aware_cache._entries" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_process_batch_responses", "qualname": "Scheduler._process_batch_responses", "full_name": "vllm_mlx.scheduler.Scheduler._process_batch_responses", "kind": "method", "signature": "def _process_batch_responses(self, responses: List[Any]) -> Tuple[List[RequestOutput], Set[str]]", "parameters": [ { "name": "responses", "kind": "positional or keyword", "annotation": "List[Any]", "default": "", "required": true, "description": "List of BatchGenerator.Response objects" } ], "return_annotation": "Tuple[List[RequestOutput], Set[str]]", "docstring": "Process responses from BatchGenerator.\n\nArgs:\n responses: List of BatchGenerator.Response objects\n\nReturns:\n Tuple of (outputs, finished_request_ids)", "summary": "Process responses from BatchGenerator.", "implementation": "Method `Scheduler._process_batch_responses` updates `self.total_completion_tokens`, `self.num_requests_processed`; calls `set`, `self.uid_to_request_id.get`, `self.running.get`, `self._store_prompt_only_cache`; returns `(outputs, finished_ids)`.", "documented": true, "public": false, "addressable": true, "line": 2583, "end_line": 2710, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2583-L2710", "decorators": [], "calls": [ "set", "self.uid_to_request_id.get", "self.running.get", "self._store_prompt_only_cache", "request.append_output_token", "_time.time", "self._get_detokenizer", "detok.add_token", "RequestOutput", "request.set_finished", "finished_ids.add", "self._detokenizer_pool.get", "detok.finalize", "self._decode_tokens", "self._cleanup_detokenizer", "hasattr", "callable", "response.prompt_cache", "self._prompt_output_entry_is_useless", "self._extract_cache_states", "logger.debug", "len", "outputs.append" ], "state_reads": [ "self.uid_to_request_id.get", "self.uid_to_request_id", "self.running.get", "self.running", "self._store_prompt_only_cache", "self._get_detokenizer", "self._detokenizer_pool.get", "self._detokenizer_pool", "self._decode_tokens", "self._cleanup_detokenizer", "self._prompt_output_entry_is_useless", "self.block_aware_cache", "self._extract_cache_states" ], "state_writes": [ "self.total_completion_tokens", "self.num_requests_processed" ], "raises": [], "return_expressions": [ "(outputs, finished_ids)" ], "awaits": false, "yields": false }, { "name": "_cleanup_finished", "qualname": "Scheduler._cleanup_finished", "full_name": "vllm_mlx.scheduler.Scheduler._cleanup_finished", "kind": "method", "signature": "def _cleanup_finished(self, finished_ids: Set[str]) -> None", "parameters": [ { "name": "finished_ids", "kind": "positional or keyword", "annotation": "Set[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Clean up finished requests and store caches for reuse.", "summary": "Clean up finished requests and store caches for reuse.", "implementation": "Method `Scheduler._cleanup_finished` calls `self.running.get`, `hasattr`, `list`, `self.block_aware_cache.store_cache`.", "documented": true, "public": false, "addressable": true, "line": 2712, "end_line": 2865, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2712-L2865", "decorators": [], "calls": [ "self.running.get", "hasattr", "list", "self.block_aware_cache.store_cache", "logger.debug", "len", "_time.monotonic", "self.memory_aware_cache.store", "logger.info", "self.prefix_cache.store_cache", "isinstance", "mx.eval", "callable", "self.finished_req_ids.add", "mx.clear_cache" ], "state_reads": [ "self.running.get", "self.running", "self.block_aware_cache", "self.block_aware_cache.store_cache", "self.memory_aware_cache", "self.memory_aware_cache.store", "self.memory_aware_cache._entries", "self.memory_aware_cache._current_memory", "self.prefix_cache", "self.prefix_cache.store_cache", "self.request_id_to_uid", "self.uid_to_request_id", "self.finished_req_ids.add", "self.finished_req_ids" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_is_cache_corruption_error", "qualname": "Scheduler._is_cache_corruption_error", "full_name": "vllm_mlx.scheduler.Scheduler._is_cache_corruption_error", "kind": "method", "signature": "def _is_cache_corruption_error(self, error: Exception) -> bool", "parameters": [ { "name": "error", "kind": "positional or keyword", "annotation": "Exception", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if an error indicates cache corruption.", "summary": "Check if an error indicates cache corruption.", "implementation": "Method `Scheduler._is_cache_corruption_error` calls `str`, `any`; returns `any((pattern in error_str for pattern in CACHE_CORRUPTION_PATTERNS))`.", "documented": true, "public": false, "addressable": true, "line": 2867, "end_line": 2870, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2867-L2870", "decorators": [], "calls": [ "str", "any" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "any((pattern in error_str for pattern in CACHE_CORRUPTION_PATTERNS))" ], "awaits": false, "yields": false }, { "name": "_is_stream_thread_error", "qualname": "Scheduler._is_stream_thread_error", "full_name": "vllm_mlx.scheduler.Scheduler._is_stream_thread_error", "kind": "method", "signature": "def _is_stream_thread_error(self, error: Exception) -> bool", "parameters": [ { "name": "error", "kind": "positional or keyword", "annotation": "Exception", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if an error indicates MLX stream/thread ownership mismatch.", "summary": "Check if an error indicates MLX stream/thread ownership mismatch.", "implementation": "Method `Scheduler._is_stream_thread_error` calls `str`; returns `'no Stream(' in error_str or 'no Stream(gpu' in error_str`.", "documented": true, "public": false, "addressable": true, "line": 2872, "end_line": 2875, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2872-L2875", "decorators": [], "calls": [ "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'no Stream(' in error_str or 'no Stream(gpu' in error_str" ], "awaits": false, "yields": false }, { "name": "_recover_from_cache_error", "qualname": "Scheduler._recover_from_cache_error", "full_name": "vllm_mlx.scheduler.Scheduler._recover_from_cache_error", "kind": "method", "signature": "def _recover_from_cache_error(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Recover from cache corruption error.", "summary": "Recover from cache corruption error.", "implementation": "Method `Scheduler._recover_from_cache_error` updates `self._current_sampler_params`; calls `self._close_batch_generator`, `self.block_aware_cache.clear`, `self.memory_aware_cache.clear`, `self.prefix_cache.clear`.", "documented": true, "public": false, "addressable": true, "line": 2877, "end_line": 2895, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2877-L2895", "decorators": [], "calls": [ "self._close_batch_generator", "self.block_aware_cache.clear", "self.memory_aware_cache.clear", "self.prefix_cache.clear", "self.request_id_to_uid.clear", "self.uid_to_request_id.clear", "logger.info" ], "state_reads": [ "self._close_batch_generator", "self.block_aware_cache", "self.block_aware_cache.clear", "self.memory_aware_cache", "self.memory_aware_cache.clear", "self.prefix_cache", "self.prefix_cache.clear", "self.request_id_to_uid.clear", "self.request_id_to_uid", "self.uid_to_request_id.clear", "self.uid_to_request_id" ], "state_writes": [ "self._current_sampler_params" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_recover_from_generation_error", "qualname": "Scheduler._recover_from_generation_error", "full_name": "vllm_mlx.scheduler.Scheduler._recover_from_generation_error", "kind": "method", "signature": "def _recover_from_generation_error(self) -> Set[str]", "parameters": [], "return_annotation": "Set[str]", "docstring": "Recover from fatal generation error (OOM, Metal crash).\n\nAborts all running requests and resets batch state.\nUnlike cache corruption recovery, does NOT reschedule —\nthe request that OOMed would just OOM again.\n\nReturns:\n Set of aborted request IDs.", "summary": "Recover from fatal generation error (OOM, Metal crash).", "implementation": "Method `Scheduler._recover_from_generation_error` updates `self._current_sampler_params`; calls `self._close_batch_generator`, `set`, `list`, `self.running.get`; returns `aborted_ids`.", "documented": true, "public": false, "addressable": true, "line": 2897, "end_line": 2933, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2897-L2933", "decorators": [], "calls": [ "self._close_batch_generator", "set", "list", "self.running.get", "request.set_finished", "aborted_ids.add", "self.finished_req_ids.add", "self.running.clear", "self._detokenizer_pool.clear", "self.request_id_to_uid.clear", "self.uid_to_request_id.clear", "mx.clear_cache", "logger.warning", "len" ], "state_reads": [ "self._close_batch_generator", "self.running", "self.running.get", "self.finished_req_ids.add", "self.finished_req_ids", "self.running.clear", "self._detokenizer_pool.clear", "self._detokenizer_pool", "self.request_id_to_uid.clear", "self.request_id_to_uid", "self.uid_to_request_id.clear", "self.uid_to_request_id" ], "state_writes": [ "self._current_sampler_params" ], "raises": [], "return_expressions": [ "aborted_ids" ], "awaits": false, "yields": false }, { "name": "_reschedule_running_requests", "qualname": "Scheduler._reschedule_running_requests", "full_name": "vllm_mlx.scheduler.Scheduler._reschedule_running_requests", "kind": "method", "signature": "def _reschedule_running_requests(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Move running requests back to waiting queue for retry.", "summary": "Move running requests back to waiting queue for retry.", "implementation": "Method `Scheduler._reschedule_running_requests` calls `len`, `list`, `self.running.items`, `self.waiting.appendleft`.", "documented": true, "public": false, "addressable": true, "line": 2935, "end_line": 2951, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2935-L2951", "decorators": [], "calls": [ "len", "list", "self.running.items", "self.waiting.appendleft", "logger.info" ], "state_reads": [ "self.running", "self.running.items", "self.waiting.appendleft", "self.waiting" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "step", "qualname": "Scheduler.step", "full_name": "vllm_mlx.scheduler.Scheduler.step", "kind": "method", "signature": "def step(self, max_retries: int=1) -> SchedulerOutput", "parameters": [ { "name": "max_retries", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Number of times to retry on cache errors (default 1)" } ], "return_annotation": "SchedulerOutput", "docstring": "Execute one scheduling step with automatic error recovery.\n\nThis method:\n1. Schedules waiting requests into the batch\n2. Runs one generation step via BatchGenerator\n3. Processes outputs and handles finished requests\n4. Automatically recovers from cache corruption errors\n\nArgs:\n max_retries: Number of times to retry on cache errors (default 1)\n\nReturns:\n SchedulerOutput with results of this step", "summary": "Execute one scheduling step with automatic error recovery.", "implementation": "Method `Scheduler.step` updates `self.finished_req_ids`, `self._step_count`; calls `SchedulerOutput`, `self._process_pending_aborts`, `range`, `self._schedule_waiting`; returns `output`.", "documented": true, "public": true, "addressable": true, "line": 2953, "end_line": 3089, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L2953-L3089", "decorators": [], "calls": [ "SchedulerOutput", "self._process_pending_aborts", "range", "self._schedule_waiting", "sum", "_sanitize_batch_generator_logits_processors", "self.batch_generator.next", "isinstance", "self._process_batch_responses", "self._cleanup_finished", "self._is_cache_corruption_error", "logger.warning", "self._recover_from_cache_error", "self._reschedule_running_requests", "logger.error", "self._is_stream_thread_error", "traceback.format_exc", "self._recover_from_generation_error", "output.outputs.append", "RequestOutput", "set", "len", "max", "hasattr", "mx.eval", "mx.clear_cache", "mx.metal.is_available", "mx.get_active_memory", "mx.get_peak_memory", "mx.get_cache_memory", "logger.info" ], "state_reads": [ "self._process_pending_aborts", "self._schedule_waiting", "self.batch_generator", "self.running", "self.batch_generator.next", "self._process_batch_responses", "self._cleanup_finished", "self._is_cache_corruption_error", "self._recover_from_cache_error", "self._reschedule_running_requests", "self._is_stream_thread_error", "self._recover_from_generation_error", "self.finished_req_ids", "self._clear_cache_interval", "self._step_count", "self.batch_generator.active_batch", "self.batch_generator.active_batch.tokens", "self._memory_log_interval", "self.waiting" ], "state_writes": [ "self.finished_req_ids", "self._step_count" ], "raises": [], "return_expressions": [ "output" ], "awaits": false, "yields": false }, { "name": "get_request", "qualname": "Scheduler.get_request", "full_name": "vllm_mlx.scheduler.Scheduler.get_request", "kind": "method", "signature": "def get_request(self, request_id: str) -> Optional[Request]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[Request]", "docstring": "Get a request by ID.", "summary": "Get a request by ID.", "implementation": "Method `Scheduler.get_request` calls `self.requests.get`; returns `self.requests.get(request_id)`.", "documented": true, "public": true, "addressable": true, "line": 3091, "end_line": 3093, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3091-L3093", "decorators": [], "calls": [ "self.requests.get" ], "state_reads": [ "self.requests.get", "self.requests" ], "state_writes": [], "raises": [], "return_expressions": [ "self.requests.get(request_id)" ], "awaits": false, "yields": false }, { "name": "remove_finished_request", "qualname": "Scheduler.remove_finished_request", "full_name": "vllm_mlx.scheduler.Scheduler.remove_finished_request", "kind": "method", "signature": "def remove_finished_request(self, request_id: str) -> Optional[Request]", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[Request]", "docstring": "Remove a finished request from tracking.", "summary": "Remove a finished request from tracking.", "implementation": "Method `Scheduler.remove_finished_request` calls `self.requests.pop`; returns `self.requests.pop(request_id, None)`.", "documented": true, "public": true, "addressable": true, "line": 3095, "end_line": 3097, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3095-L3097", "decorators": [], "calls": [ "self.requests.pop" ], "state_reads": [ "self.requests.pop", "self.requests" ], "state_writes": [], "raises": [], "return_expressions": [ "self.requests.pop(request_id, None)" ], "awaits": false, "yields": false }, { "name": "get_running_requests_info", "qualname": "Scheduler.get_running_requests_info", "full_name": "vllm_mlx.scheduler.Scheduler.get_running_requests_info", "kind": "method", "signature": "def get_running_requests_info(self) -> List[Dict[str, Any]]", "parameters": [], "return_annotation": "List[Dict[str, Any]]", "docstring": "Per-request details for status endpoint.", "summary": "Per-request details for status endpoint.", "implementation": "Method `Scheduler.get_running_requests_info` calls `_time.time`, `result.append`, `round`, `self.running.values`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 3099, "end_line": 3165, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3099-L3165", "decorators": [], "calls": [ "_time.time", "result.append", "round", "self.running.values", "min" ], "state_reads": [ "self.waiting", "self.running.values", "self.running" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "Scheduler.get_stats", "full_name": "vllm_mlx.scheduler.Scheduler.get_stats", "kind": "method", "signature": "def get_stats(self) -> Dict[str, Any]", "parameters": [], "return_annotation": "Dict[str, Any]", "docstring": "Get scheduler statistics.", "summary": "Get scheduler statistics.", "implementation": "Method `Scheduler.get_stats` calls `len`, `stats.update`, `_mtp_status_snapshot`, `mx.metal.is_available`; returns `stats`.", "documented": true, "public": true, "addressable": true, "line": 3167, "end_line": 3193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3167-L3193", "decorators": [], "calls": [ "len", "stats.update", "_mtp_status_snapshot", "mx.metal.is_available", "round", "mx.get_active_memory", "mx.get_peak_memory", "mx.get_cache_memory", "self.block_aware_cache.get_stats", "self.memory_aware_cache.get_stats", "self.prefix_cache.get_stats" ], "state_reads": [ "self.waiting", "self.running", "self.num_requests_processed", "self.total_prompt_tokens", "self.total_completion_tokens", "self.batch_generator", "self.block_aware_cache", "self.block_aware_cache.get_stats", "self.memory_aware_cache", "self.memory_aware_cache.get_stats", "self.prefix_cache", "self.prefix_cache.get_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "stats" ], "awaits": false, "yields": false }, { "name": "get_cache_stats", "qualname": "Scheduler.get_cache_stats", "full_name": "vllm_mlx.scheduler.Scheduler.get_cache_stats", "kind": "method", "signature": "def get_cache_stats(self) -> Optional[Dict[str, Any]]", "parameters": [], "return_annotation": "Optional[Dict[str, Any]]", "docstring": "Get cache statistics.", "summary": "Get cache statistics.", "implementation": "Method `Scheduler.get_cache_stats` calls `self.block_aware_cache.get_stats`, `self.memory_aware_cache.get_stats`, `self.prefix_cache.get_stats`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3195, "end_line": 3203, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3195-L3203", "decorators": [], "calls": [ "self.block_aware_cache.get_stats", "self.memory_aware_cache.get_stats", "self.prefix_cache.get_stats" ], "state_reads": [ "self.block_aware_cache", "self.block_aware_cache.get_stats", "self.memory_aware_cache", "self.memory_aware_cache.get_stats", "self.prefix_cache", "self.prefix_cache.get_stats" ], "state_writes": [], "raises": [], "return_expressions": [ "self.block_aware_cache.get_stats()", "self.memory_aware_cache.get_stats()", "self.prefix_cache.get_stats()", "None" ], "awaits": false, "yields": false }, { "name": "clear_runtime_caches", "qualname": "Scheduler.clear_runtime_caches", "full_name": "vllm_mlx.scheduler.Scheduler.clear_runtime_caches", "kind": "method", "signature": "def clear_runtime_caches(self) -> Dict[str, bool]", "parameters": [], "return_annotation": "Dict[str, bool]", "docstring": "Clear prefix-cache state without resetting scheduler/request state.", "summary": "Clear prefix-cache state without resetting scheduler/request state.", "implementation": "Method `Scheduler.clear_runtime_caches` calls `self.block_aware_cache.clear`, `self.memory_aware_cache.clear`, `self.prefix_cache.clear`; returns `cleared`.", "documented": true, "public": true, "addressable": true, "line": 3205, "end_line": 3221, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3205-L3221", "decorators": [], "calls": [ "self.block_aware_cache.clear", "self.memory_aware_cache.clear", "self.prefix_cache.clear" ], "state_reads": [ "self.block_aware_cache", "self.block_aware_cache.clear", "self.memory_aware_cache", "self.memory_aware_cache.clear", "self.prefix_cache", "self.prefix_cache.clear" ], "state_writes": [], "raises": [], "return_expressions": [ "cleared" ], "awaits": false, "yields": false }, { "name": "reset", "qualname": "Scheduler.reset", "full_name": "vllm_mlx.scheduler.Scheduler.reset", "kind": "method", "signature": "def reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset the scheduler state.", "summary": "Reset the scheduler state.", "implementation": "Method `Scheduler.reset` updates `self._current_sampler_params`; calls `self._pending_abort_ids.clear`, `list`, `self.requests.keys`, `self._do_abort_request`.", "documented": true, "public": true, "addressable": true, "line": 3223, "end_line": 3246, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3223-L3246", "decorators": [], "calls": [ "self._pending_abort_ids.clear", "list", "self.requests.keys", "self._do_abort_request", "self.waiting.clear", "self.running.clear", "self.requests.clear", "self.finished_req_ids.clear", "self.request_id_to_uid.clear", "self.uid_to_request_id.clear", "self._detokenizer_pool.clear", "self._close_batch_generator", "self.clear_runtime_caches", "self.close_ssd_tier" ], "state_reads": [ "self._pending_abort_ids.clear", "self._pending_abort_ids", "self.requests.keys", "self.requests", "self._do_abort_request", "self.waiting.clear", "self.waiting", "self.running.clear", "self.running", "self.requests.clear", "self.finished_req_ids.clear", "self.finished_req_ids", "self.request_id_to_uid.clear", "self.request_id_to_uid", "self.uid_to_request_id.clear", "self.uid_to_request_id", "self._detokenizer_pool.clear", "self._detokenizer_pool", "self._close_batch_generator", "self.clear_runtime_caches", "self.close_ssd_tier" ], "state_writes": [ "self._current_sampler_params" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "deep_reset", "qualname": "Scheduler.deep_reset", "full_name": "vllm_mlx.scheduler.Scheduler.deep_reset", "kind": "method", "signature": "def deep_reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Deep reset that clears ALL cache state including model-level caches.\n\nThis is more aggressive than reset() and should be used when\nswitching engines or recovering from errors.", "summary": "Deep reset that clears ALL cache state including model-level caches.", "implementation": "Method `Scheduler.deep_reset` updates `self.model.cache`; calls `self.reset`, `hasattr`, `gc.collect`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 3248, "end_line": 3276, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3248-L3276", "decorators": [], "calls": [ "self.reset", "hasattr", "gc.collect", "logger.info" ], "state_reads": [ "self.reset", "self.model", "self.model.layers" ], "state_writes": [ "self.model.cache" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "save_cache_to_disk", "qualname": "Scheduler.save_cache_to_disk", "full_name": "vllm_mlx.scheduler.Scheduler.save_cache_to_disk", "kind": "method", "signature": "def save_cache_to_disk(self, cache_dir: str) -> bool", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Save prefix cache to disk for persistence across restarts.", "summary": "Save prefix cache to disk for persistence across restarts.", "implementation": "Method `Scheduler.save_cache_to_disk` calls `self.memory_aware_cache.save_to_disk`, `logger.info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3282, "end_line": 3287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3282-L3287", "decorators": [], "calls": [ "self.memory_aware_cache.save_to_disk", "logger.info" ], "state_reads": [ "self.memory_aware_cache", "self.memory_aware_cache.save_to_disk" ], "state_writes": [], "raises": [], "return_expressions": [ "self.memory_aware_cache.save_to_disk(cache_dir)", "False" ], "awaits": false, "yields": false }, { "name": "load_cache_from_disk", "qualname": "Scheduler.load_cache_from_disk", "full_name": "vllm_mlx.scheduler.Scheduler.load_cache_from_disk", "kind": "method", "signature": "def load_cache_from_disk(self, cache_dir: str) -> int", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Load prefix cache from disk. Returns number of entries loaded.", "summary": "Load prefix cache from disk.", "implementation": "Method `Scheduler.load_cache_from_disk` calls `self.memory_aware_cache.load_from_disk`, `logger.info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3289, "end_line": 3294, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3289-L3294", "decorators": [], "calls": [ "self.memory_aware_cache.load_from_disk", "logger.info" ], "state_reads": [ "self.memory_aware_cache", "self.memory_aware_cache.load_from_disk" ], "state_writes": [], "raises": [], "return_expressions": [ "self.memory_aware_cache.load_from_disk(cache_dir)", "0" ], "awaits": false, "yields": false }, { "name": "clear_prefix_cache", "qualname": "Scheduler.clear_prefix_cache", "full_name": "vllm_mlx.scheduler.Scheduler.clear_prefix_cache", "kind": "method", "signature": "def clear_prefix_cache(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear the in-memory prefix cache (keeps disk cache untouched).", "summary": "Clear the in-memory prefix cache (keeps disk cache untouched).", "implementation": "Method `Scheduler.clear_prefix_cache` calls `hasattr`, `self.memory_aware_cache.clear`, `logger.info`, `self.prefix_cache.clear`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 3296, "end_line": 3306, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3296-L3306", "decorators": [], "calls": [ "hasattr", "self.memory_aware_cache.clear", "logger.info", "self.prefix_cache.clear" ], "state_reads": [ "self.memory_aware_cache", "self.memory_aware_cache.clear", "self.prefix_cache", "self.prefix_cache.clear" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "close_ssd_tier", "qualname": "Scheduler.close_ssd_tier", "full_name": "vllm_mlx.scheduler.Scheduler.close_ssd_tier", "kind": "method", "signature": "def close_ssd_tier(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Shut down the SSD cache tier if present.", "summary": "Shut down the SSD cache tier if present.", "implementation": "Method `Scheduler.close_ssd_tier` updates `self._ssd_tier`; calls `self._ssd_tier.close`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 3308, "end_line": 3313, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3308-L3313", "decorators": [], "calls": [ "self._ssd_tier.close", "logger.info" ], "state_reads": [ "self._ssd_tier", "self._ssd_tier.close" ], "state_writes": [ "self._ssd_tier" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_try_promote_ssd_pending", "qualname": "Scheduler._try_promote_ssd_pending", "full_name": "vllm_mlx.scheduler.Scheduler._try_promote_ssd_pending", "kind": "method", "signature": "def _try_promote_ssd_pending(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Attempt synchronous SSD promotion for waiting requests tagged ssd_pending.\n\nCalled from _schedule_waiting() before requests are moved to running.\nReads SSD entries synchronously (disk I/O stays out of fetch() per spec).", "summary": "Attempt synchronous SSD promotion for waiting requests tagged ssd_pending.", "implementation": "Method `Scheduler._try_promote_ssd_pending` updates `self._ssd_tier._stats.promotion_failures`, `self._ssd_tier._stats.ssd_hits`; calls `getattr`, `self.memory_aware_cache.try_reserve_memory`, `logger.info`, `tuple`.", "documented": true, "public": false, "addressable": true, "line": 3315, "end_line": 3395, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3315-L3395", "decorators": [], "calls": [ "getattr", "self.memory_aware_cache.try_reserve_memory", "logger.info", "tuple", "self._ssd_tier._read_entry", "self.memory_aware_cache.release_reserved_memory", "logger.exception", "self._reconstruct_ssd_layers", "self.memory_aware_cache.store", "list", "self._ssd_tier._index.touch", "len" ], "state_reads": [ "self.waiting", "self.memory_aware_cache", "self.memory_aware_cache.try_reserve_memory", "self._ssd_tier._stats", "self._ssd_tier", "self._ssd_tier._read_entry", "self.memory_aware_cache.release_reserved_memory", "self._reconstruct_ssd_layers", "self.memory_aware_cache.store", "self._ssd_tier._index.touch", "self._ssd_tier._index" ], "state_writes": [ "self._ssd_tier._stats.promotion_failures", "self._ssd_tier._stats.ssd_hits" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "promote_from_ssd", "qualname": "Scheduler.promote_from_ssd", "full_name": "vllm_mlx.scheduler.Scheduler.promote_from_ssd", "kind": "method", "signature": "async def promote_from_ssd(self, request) -> bool", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Promote a cold-tier cache entry for a request (async version).\n\nAlternative to _try_promote_ssd_pending() for callers with an\nasync event loop. Uses asyncio.to_thread for non-blocking disk I/O.\n\nReturns True if promotion succeeded and request was updated.", "summary": "Promote a cold-tier cache entry for a request (async version).", "implementation": "Method `Scheduler.promote_from_ssd` calls `getattr`, `candidate.get`, `len`, `tuple`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3397, "end_line": 3460, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3397-L3460", "decorators": [], "calls": [ "getattr", "candidate.get", "len", "tuple", "self._ssd_tier.async_promote", "release_budget", "self._reconstruct_ssd_layers", "self.memory_aware_cache.store", "list", "logger.info" ], "state_reads": [ "self._ssd_tier", "self._ssd_tier.async_promote", "self._reconstruct_ssd_layers", "self.memory_aware_cache.store", "self.memory_aware_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": true, "yields": false }, { "name": "reserve_budget", "qualname": "Scheduler.promote_from_ssd.reserve_budget", "full_name": "vllm_mlx.scheduler.Scheduler.promote_from_ssd.reserve_budget", "kind": "nested function", "signature": "def reserve_budget(nbytes: int) -> bool", "parameters": [ { "name": "nbytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Tentatively reserve RAM budget for promotion.", "summary": "Tentatively reserve RAM budget for promotion.", "implementation": "Nested Function `Scheduler.promote_from_ssd.reserve_budget` calls `self.memory_aware_cache.try_reserve_memory`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 3412, "end_line": 3416, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3412-L3416", "decorators": [], "calls": [ "self.memory_aware_cache.try_reserve_memory" ], "state_reads": [ "self.memory_aware_cache", "self.memory_aware_cache.try_reserve_memory" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "self.memory_aware_cache.try_reserve_memory(nbytes)" ], "awaits": false, "yields": false }, { "name": "release_budget", "qualname": "Scheduler.promote_from_ssd.release_budget", "full_name": "vllm_mlx.scheduler.Scheduler.promote_from_ssd.release_budget", "kind": "nested function", "signature": "def release_budget(nbytes: int) -> None", "parameters": [ { "name": "nbytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Release tentatively reserved budget on failure.", "summary": "Release tentatively reserved budget on failure.", "implementation": "Nested Function `Scheduler.promote_from_ssd.release_budget` calls `self.memory_aware_cache.release_reserved_memory`.", "documented": true, "public": false, "addressable": false, "line": 3418, "end_line": 3421, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3418-L3421", "decorators": [], "calls": [ "self.memory_aware_cache.release_reserved_memory" ], "state_reads": [ "self.memory_aware_cache", "self.memory_aware_cache.release_reserved_memory" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_reconstruct_ssd_layers", "qualname": "Scheduler._reconstruct_ssd_layers", "full_name": "vllm_mlx.scheduler.Scheduler._reconstruct_ssd_layers", "kind": "method", "signature": "def _reconstruct_ssd_layers(self, layer_dicts: list[dict]) -> list | None", "parameters": [ { "name": "layer_dicts", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list | None", "docstring": "Reconstruct cache objects from deserialized layer dicts.\n\nConverts numpy arrays back to MLX arrays and creates KVCache objects.", "summary": "Reconstruct cache objects from deserialized layer dicts.", "implementation": "Method `Scheduler._reconstruct_ssd_layers` calls `KVCache`, `mx.array`, `ld.get`, `_mx_dtype_from_name`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 3462, "end_line": 3518, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3462-L3518", "decorators": [], "calls": [ "KVCache", "mx.array", "ld.get", "_mx_dtype_from_name", "kv.keys.astype", "kv.values.astype", "setattr", "result.append", "enumerate", "state_arrays[i].astype", "ArraysCache", "len", "logger.warning", "list", "ld.keys" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "result" ], "awaits": false, "yields": false }, { "name": "_mx_dtype_from_name", "qualname": "Scheduler._reconstruct_ssd_layers._mx_dtype_from_name", "full_name": "vllm_mlx.scheduler.Scheduler._reconstruct_ssd_layers._mx_dtype_from_name", "kind": "nested function", "signature": "def _mx_dtype_from_name(name: str)", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `Scheduler._reconstruct_ssd_layers._mx_dtype_from_name` calls `getattr`; returns `getattr(mx, name, None)`.", "implementation": "Nested Function `Scheduler._reconstruct_ssd_layers._mx_dtype_from_name` calls `getattr`; returns `getattr(mx, name, None)`.", "documented": false, "public": false, "addressable": false, "line": 3473, "end_line": 3474, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/scheduler.py#L3473-L3474", "decorators": [], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "getattr(mx, name, None)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.server", "path": "vllm_mlx/server.py", "page_path": "reference/api/vllm_mlx/server.md", "docstring": "Unified OpenAI-compatible API server for vllm-mlx.\n\nThis module provides a FastAPI server that exposes an OpenAI-compatible\nAPI for LLM and MLLM (Multimodal Language Model) inference using MLX on Apple Silicon.\n\nSupports two modes:\n- Simple mode (default): Maximum throughput for single-user scenarios\n- Batched mode: Continuous batching for multiple concurrent users\n\nFeatures:\n- Text-only LLM inference (mlx-lm)\n- Multimodal MLLM inference with images and video (mlx-vlm)\n- OpenAI-compatible chat/completions API\n- Streaming responses\n- MCP (Model Context Protocol) tool integration\n- Tool calling (Qwen/Llama formats)\n\nUsage:\n # Simple mode (maximum throughput)\n python -m vllm_mlx.server --model mlx-community/Llama-3.2-3B-Instruct-4bit\n\n # Batched mode (for multiple concurrent users)\n python -m vllm_mlx.server --model mlx-community/Llama-3.2-3B-Instruct-4bit --continuous-batching\n\n # With MCP tools\n python -m vllm_mlx.server --model mlx-community/Qwen3-4B-4bit --mcp-config mcp.json\n\nThe server provides:\n - POST /v1/completions - Text completions\n - POST /v1/chat/completions - Chat completions (with multimodal support)\n - GET /v1/models - List available models\n - GET /health - Health check\n - GET /v1/mcp/tools - List MCP tools\n - GET /v1/mcp/servers - MCP server status\n - POST /v1/mcp/execute - Execute MCP tool", "summary": "Unified OpenAI-compatible API server for vllm-mlx.", "line_count": 6916, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1-L6916", "members": [ "logger", "_IMPORTED_SIMPLE_ENGINE", "_engine", "_model_manager", "_model_name", "_model_path", "_warm_prompts_path", "_default_model_key", "_default_max_tokens", "_max_request_tokens", "_default_timeout", "_default_temperature", "_default_top_p", "_default_chat_template_kwargs", "_default_top_k", "_default_min_p", "_default_presence_penalty", "_default_repetition_penalty", "_metrics_enabled", "_max_audio_upload_bytes", "_max_tts_input_chars", "_force_mllm_model", "_default_thinking_token_budget", "_auto_unload_idle_seconds", "_lazy_load_model", "_residency_manager", "_lifecycle_task", "_lifespan_active", "_FALLBACK_TEMPERATURE", "_FALLBACK_TOP_P", "_FALLBACK_TOP_K", "_FALLBACK_MIN_P", "_FALLBACK_PRESENCE_PENALTY", "_FALLBACK_REPETITION_PENALTY", "_resolve_temperature", "_resolve_top_p", "_resolve_top_k", "_resolve_min_p", "_resolve_presence_penalty", "_resolve_repetition_penalty", "_resolve_request_max_tokens", "_resolve_chat_template_kwargs", "PreparedChatInvocation", "_prepare_chat_messages", "_iter_remote_media_urls", "_validate_remote_media_urls", "_raise_remote_media_http_error", "_prepare_json_logits_processor", "_build_thinking_processor", "_resolve_no_final_content_token_limit", "_generation_metadata", "_ThinkingAwareLogitsProcessor", "_attach_response_format_logits_processor", "_coerce_logit_bias", "_attach_logit_bias_processor", "_prepare_chat_completion_invocation", "_prepare_anthropic_invocation", "_mcp_manager", "_mcp_executor", "_embedding_engine", "_embedding_model_locked", "_rerank_engine", "_rerank_model_locked", "_api_key", "_auth_warning_logged", "_reasoning_parser", "_reasoning_parser_name", "_thinking_disabled", "_enable_auto_tool_choice", "_tool_call_parser", "_tool_parser_instance", "_responses_store", "_RESPONSES_STORE_MAX_SIZE", "_TOOL_MARKUP_PATTERN", "_STREAMING_TOOL_MARKERS", "_STREAMING_BARE_BRACKET_MARKER", "_STREAMING_BARE_BRACKET_PARTIAL", "_STREAMING_TOOL_MARKUP_SCAN_CHARS", "_strip_backslash_before_unicode", "_sanitize_log_text", "_log_and_raise_internal_error", "_raise_engine_busy", "RequestModelContext", "_list_available_model_names", "_response_model_name", "_acquire_request_model", "_stream_with_model_context", "_build_tool_parser", "_build_reasoning_parser", "_prepare_streaming_reasoning_parser", "_prepare_openai_stream_reasoning_state", "_request_tool_definitions", "_streaming_json_fence_stripper", "_idle_unload_enabled", "_get_idle_unload_event", "_invalidate_tool_parser_cache", "_load_prefix_cache_from_disk", "_save_prefix_cache_to_disk", "_get_cache_dir", "_build_engine", "_engine_factory", "_run_blocking_engine_cache_io", "_restore_engine_state", "_persist_engine_state", "_activate_engine", "_sync_engine_from_residency", "_get_lifecycle_status", "_public_lifecycle_status", "_lifecycle_loop", "_acquire_default_engine", "_release_default_engine", "lifespan", "app", "security", "_metrics_result_from_status", "_metrics_path_for_request", "_metrics_middleware", "RateLimiter", "_rate_limiter", "check_rate_limit", "verify_api_key", "get_engine", "_coerce_tool_arguments", "_validate_model_name", "_get_engine_tokenizer", "_get_or_init_tool_parser", "_parse_tool_calls_with_parser", "_apply_response_format_or_raise", "_response_format_type", "_promote_streaming_response_format_delta", "_new_response_item_id", "_response_content_to_text", "_responses_tools_to_chat_tools", "_responses_input_to_chat_messages", "_responses_request_to_new_persisted_messages", "_responses_request_to_persisted_messages", "_responses_request_to_chat_request", "_build_responses_output_items", "_response_output_items_to_chat_messages", "_build_response_object", "_prepare_responses_request", "_prepare_streaming_responses_request", "_run_responses_request", "_stream_responses_request", "_responses_sse_event", "_HARMONY_ANALYSIS_BLOCK_RE", "_strip_harmony_analysis_blocks", "_extract_reasoning_and_tool_calls", "_detect_native_tool_support", "_detect_harmony_rendering", "_tool_choice_disabled", "_get_streaming_tool_parser", "_extract_streaming_tool_delta", "_stream_request_metadata", "_parse_streaming_tool_content", "_streaming_tool_markup_possible", "_streaming_tool_markup_possible_after_delta", "load_embedding_model", "load_reranker_model", "load_model", "load_model_registry", "get_usage", "metrics", "health", "status", "cache_stats", "clear_cache", "clear_prefix_cache", "cancel_request", "delete_request", "list_models", "create_embeddings", "rerank_documents", "list_mcp_tools", "list_mcp_servers", "execute_mcp_tool", "_stt_engine", "_tts_engine", "create_transcription", "create_speech", "list_voices", "_ensure_sse_terminal", "_find_uvicorn_cycle", "_is_client_disconnected", "_disconnect_guard", "_wait_with_disconnect", "_start_request_budget", "_remaining_request_timeout", "_active_request_contexts", "_acquire_default_engine_for_request", "_release_engine_for_request", "_make_release_cleanup", "create_completion", "create_chat_completion", "_normalize_messages", "create_response", "_get_forced_tool_name", "_apply_forced_tool_choice", "_tool_name", "_inject_json_instruction", "_convert_anthropic_stop_reason", "_prepare_anthropic_endpoint_invocation", "create_anthropic_message", "count_anthropic_tokens", "_emit_content_pieces", "_stream_anthropic_messages", "stream_completion", "stream_chat_completion", "init_mcp", "_make_keepalive_http_protocol", "main", "create_parser" ], "symbols": [ { "name": "_resolve_temperature", "qualname": "_resolve_temperature", "full_name": "vllm_mlx.server._resolve_temperature", "kind": "function", "signature": "def _resolve_temperature(request_value: float | None) -> float", "parameters": [ { "name": "request_value", "kind": "positional or keyword", "annotation": "float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Resolve temperature: request > CLI default > fallback.", "summary": "Resolve temperature: request > CLI default > fallback.", "implementation": "Function `_resolve_temperature` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 225, "end_line": 231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L225-L231", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "request_value", "_default_temperature", "_FALLBACK_TEMPERATURE" ], "awaits": false, "yields": false }, { "name": "_resolve_top_p", "qualname": "_resolve_top_p", "full_name": "vllm_mlx.server._resolve_top_p", "kind": "function", "signature": "def _resolve_top_p(request_value: float | None) -> float", "parameters": [ { "name": "request_value", "kind": "positional or keyword", "annotation": "float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Resolve top_p: request > CLI default > fallback.", "summary": "Resolve top_p: request > CLI default > fallback.", "implementation": "Function `_resolve_top_p` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 234, "end_line": 240, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L234-L240", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "request_value", "_default_top_p", "_FALLBACK_TOP_P" ], "awaits": false, "yields": false }, { "name": "_resolve_top_k", "qualname": "_resolve_top_k", "full_name": "vllm_mlx.server._resolve_top_k", "kind": "function", "signature": "def _resolve_top_k(request_value: int | None) -> int", "parameters": [ { "name": "request_value", "kind": "positional or keyword", "annotation": "int | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Resolve top_k: request > CLI default > fallback.", "summary": "Resolve top_k: request > CLI default > fallback.", "implementation": "Function `_resolve_top_k` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 243, "end_line": 249, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L243-L249", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "request_value", "_default_top_k", "_FALLBACK_TOP_K" ], "awaits": false, "yields": false }, { "name": "_resolve_min_p", "qualname": "_resolve_min_p", "full_name": "vllm_mlx.server._resolve_min_p", "kind": "function", "signature": "def _resolve_min_p(request_value: float | None) -> float", "parameters": [ { "name": "request_value", "kind": "positional or keyword", "annotation": "float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Resolve min_p: request > CLI default > fallback.", "summary": "Resolve min_p: request > CLI default > fallback.", "implementation": "Function `_resolve_min_p` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 252, "end_line": 258, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L252-L258", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "request_value", "_default_min_p", "_FALLBACK_MIN_P" ], "awaits": false, "yields": false }, { "name": "_resolve_presence_penalty", "qualname": "_resolve_presence_penalty", "full_name": "vllm_mlx.server._resolve_presence_penalty", "kind": "function", "signature": "def _resolve_presence_penalty(request_value: float | None) -> float", "parameters": [ { "name": "request_value", "kind": "positional or keyword", "annotation": "float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Resolve presence_penalty: request > CLI default > fallback.", "summary": "Resolve presence_penalty: request > CLI default > fallback.", "implementation": "Function `_resolve_presence_penalty` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 261, "end_line": 267, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L261-L267", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "request_value", "_default_presence_penalty", "_FALLBACK_PRESENCE_PENALTY" ], "awaits": false, "yields": false }, { "name": "_resolve_repetition_penalty", "qualname": "_resolve_repetition_penalty", "full_name": "vllm_mlx.server._resolve_repetition_penalty", "kind": "function", "signature": "def _resolve_repetition_penalty(request_value: float | None) -> float", "parameters": [ { "name": "request_value", "kind": "positional or keyword", "annotation": "float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Resolve repetition_penalty: request > CLI default > fallback.", "summary": "Resolve repetition_penalty: request > CLI default > fallback.", "implementation": "Function `_resolve_repetition_penalty` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 270, "end_line": 276, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L270-L276", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "request_value", "_default_repetition_penalty", "_FALLBACK_REPETITION_PENALTY" ], "awaits": false, "yields": false }, { "name": "_resolve_request_max_tokens", "qualname": "_resolve_request_max_tokens", "full_name": "vllm_mlx.server._resolve_request_max_tokens", "kind": "function", "signature": "def _resolve_request_max_tokens(requested_value: int | None) -> int", "parameters": [ { "name": "requested_value", "kind": "positional or keyword", "annotation": "int | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Resolve and validate a request's max_tokens budget.", "summary": "Resolve and validate a request's max_tokens budget.", "implementation": "Function `_resolve_request_max_tokens` calls `HTTPException`; can raise `HTTPException`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 279, "end_line": 288, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L279-L288", "decorators": [], "calls": [ "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "_default_max_tokens", "requested_value" ], "awaits": false, "yields": false }, { "name": "_resolve_chat_template_kwargs", "qualname": "_resolve_chat_template_kwargs", "full_name": "vllm_mlx.server._resolve_chat_template_kwargs", "kind": "function", "signature": "def _resolve_chat_template_kwargs(request_value: dict[str, object] | None) -> dict[str, object]", "parameters": [ { "name": "request_value", "kind": "positional or keyword", "annotation": "dict[str, object] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, object]", "docstring": "Resolve chat template kwargs: request > server default > empty dict.", "summary": "Resolve chat template kwargs: request > server default > empty dict.", "implementation": "Function `_resolve_chat_template_kwargs` calls `resolved.update`; returns `resolved`.", "documented": true, "public": false, "addressable": true, "line": 291, "end_line": 300, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L291-L300", "decorators": [], "calls": [ "resolved.update" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "resolved" ], "awaits": false, "yields": false }, { "name": "PreparedChatInvocation", "qualname": "PreparedChatInvocation", "full_name": "vllm_mlx.server.PreparedChatInvocation", "kind": "class", "signature": "class PreparedChatInvocation", "parameters": [ { "name": "messages", "kind": "field", "annotation": "list[dict]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "chat_kwargs", "kind": "field", "annotation": "dict[str, object]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "response_format", "kind": "field", "annotation": "object | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "json_logits_processor", "kind": "field", "annotation": "object | None", "default": "", "required": true, "description": "Required constructor field." }, { "name": "thinking_processor", "kind": "field", "annotation": "object | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "PreparedChatInvocation", "docstring": "Fully prepared inputs for a single engine.chat/stream_chat call.", "summary": "Fully prepared inputs for a single engine.chat/stream_chat call.", "implementation": "Class `PreparedChatInvocation` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 304, "end_line": 311, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L304-L311", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_prepare_chat_messages", "qualname": "_prepare_chat_messages", "full_name": "vllm_mlx.server._prepare_chat_messages", "kind": "function", "signature": "def _prepare_chat_messages(engine: BaseEngine, request_messages: list[Message | dict]) -> tuple[list[dict], list, list, list, bool]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request_messages", "kind": "positional or keyword", "annotation": "list[Message | dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[list[dict], list, list, list, bool]", "docstring": "Normalize messages and collect media once for both stream/non-stream paths.", "summary": "Normalize messages and collect media once for both stream/non-stream paths.", "implementation": "Function `_prepare_chat_messages` calls `_validate_remote_media_urls`, `bool`, `getattr`, `hasattr`; returns `(messages, images, videos, audios, has_media)`.", "documented": true, "public": false, "addressable": true, "line": 314, "end_line": 398, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L314-L398", "decorators": [], "calls": [ "_validate_remote_media_urls", "bool", "getattr", "hasattr", "msg.model_dump", "dict", "raw.items", "messages.append", "logger.debug", "len", "msg_dict.get", "tc.get", "func.get", "isinstance", "json.loads", "_normalize_messages", "extract_multimodal_content", "canonicalize_system_messages", "msg.get", "item.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(messages, images, videos, audios, has_media)" ], "awaits": false, "yields": false }, { "name": "_iter_remote_media_urls", "qualname": "_iter_remote_media_urls", "full_name": "vllm_mlx.server._iter_remote_media_urls", "kind": "function", "signature": "def _iter_remote_media_urls(messages: list[Message | dict])", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[Message | dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Yield remote media URLs from OpenAI-style multimodal message content.", "summary": "Yield remote media URLs from OpenAI-style multimodal message content.", "implementation": "Function `_iter_remote_media_urls` calls `isinstance`, `msg.get`, `hasattr`, `item.model_dump`; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 401, "end_line": 429, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L401-L429", "decorators": [], "calls": [ "isinstance", "msg.get", "hasattr", "item.model_dump", "item.dict().items", "item.dict", "item.get", "media_value.get", "is_url" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "_validate_remote_media_urls", "qualname": "_validate_remote_media_urls", "full_name": "vllm_mlx.server._validate_remote_media_urls", "kind": "function", "signature": "def _validate_remote_media_urls(messages: list[Message | dict]) -> None", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[Message | dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Validate remote media URLs during request preparation.", "summary": "Validate remote media URLs during request preparation.", "implementation": "Function `_validate_remote_media_urls` calls `_iter_remote_media_urls`, `_validate_url_safety`.", "documented": true, "public": false, "addressable": true, "line": 432, "end_line": 435, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L432-L435", "decorators": [], "calls": [ "_iter_remote_media_urls", "_validate_url_safety" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_raise_remote_media_http_error", "qualname": "_raise_remote_media_http_error", "full_name": "vllm_mlx.server._raise_remote_media_http_error", "kind": "function", "signature": "def _raise_remote_media_http_error(exc: UnsafeRemoteURLError) -> None", "parameters": [ { "name": "exc", "kind": "positional or keyword", "annotation": "UnsafeRemoteURLError", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Log internal URL-safety detail while returning a generic client error.", "summary": "Log internal URL-safety detail while returning a generic client error.", "implementation": "Function `_raise_remote_media_http_error` calls `logger.warning`, `_sanitize_log_text`, `HTTPException`; can raise `HTTPException`.", "documented": true, "public": false, "addressable": true, "line": 438, "end_line": 444, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L438-L444", "decorators": [], "calls": [ "logger.warning", "_sanitize_log_text", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_prepare_json_logits_processor", "qualname": "_prepare_json_logits_processor", "full_name": "vllm_mlx.server._prepare_json_logits_processor", "kind": "function", "signature": "def _prepare_json_logits_processor(engine: BaseEngine, messages: list[dict], response_format: object | None, *, tools: list | None, tool_choice: object | None, log_context: str | None=None, thinking_model: bool=False) -> tuple[list[dict], object | None]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "response_format", "kind": "positional or keyword", "annotation": "object | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tools", "kind": "keyword-only", "annotation": "list | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "tool_choice", "kind": "keyword-only", "annotation": "object | None", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "log_context", "kind": "keyword-only", "annotation": "str | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "thinking_model", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." } ], "return_annotation": "tuple[list[dict], object | None]", "docstring": "Inject response_format instruction and build constrained decoding processor.", "summary": "Inject response_format instruction and build constrained decoding processor.", "implementation": "Function `_prepare_json_logits_processor` calls `build_json_system_prompt`, `_inject_json_instruction`, `_get_engine_tokenizer`, `build_json_logits_processor`; returns `(messages, json_logits_processor)`.", "documented": true, "public": false, "addressable": true, "line": 447, "end_line": 497, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L447-L497", "decorators": [], "calls": [ "build_json_system_prompt", "_inject_json_instruction", "_get_engine_tokenizer", "build_json_logits_processor", "logger.warning", "logger.info", "isinstance", "getattr", "response_format.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(messages, json_logits_processor)" ], "awaits": false, "yields": false }, { "name": "_build_thinking_processor", "qualname": "_build_thinking_processor", "full_name": "vllm_mlx.server._build_thinking_processor", "kind": "function", "signature": "def _build_thinking_processor(engine: BaseEngine, thinking_token_budget: int, *, inner: object | None=None, prompt_has_think_tag: bool=True) -> object | None", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "thinking_token_budget", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "inner", "kind": "keyword-only", "annotation": "object | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "prompt_has_think_tag", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "object | None", "docstring": "Build a ThinkingAwareLogitsProcessor if the tokenizer has think tokens.", "summary": "Build a ThinkingAwareLogitsProcessor if the tokenizer has think tokens.", "implementation": "Function `_build_thinking_processor` calls `_get_engine_tokenizer`, `tokenizer.encode`, `logger.debug`, `getattr`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 500, "end_line": 554, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L500-L554", "decorators": [], "calls": [ "_get_engine_tokenizer", "tokenizer.encode", "logger.debug", "getattr", "_resolve_no_final_content_token_limit", "logger.warning", "ThinkingAwareLogitsProcessor", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "proc" ], "awaits": false, "yields": false }, { "name": "_resolve_no_final_content_token_limit", "qualname": "_resolve_no_final_content_token_limit", "full_name": "vllm_mlx.server._resolve_no_final_content_token_limit", "kind": "function", "signature": "def _resolve_no_final_content_token_limit() -> int | None", "parameters": [], "return_annotation": "int | None", "docstring": "", "summary": "Function `_resolve_no_final_content_token_limit` calls `os.environ.get`, `raw.strip`, `int`, `logger.warning`; has 2 explicit return paths.", "implementation": "Function `_resolve_no_final_content_token_limit` calls `os.environ.get`, `raw.strip`, `int`, `logger.warning`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 557, "end_line": 568, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L557-L568", "decorators": [], "calls": [ "os.environ.get", "raw.strip", "int", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "value" ], "awaits": false, "yields": false }, { "name": "_generation_metadata", "qualname": "_generation_metadata", "full_name": "vllm_mlx.server._generation_metadata", "kind": "function", "signature": "def _generation_metadata(thinking_processor: object | None) -> GenerationMetadata | None", "parameters": [ { "name": "thinking_processor", "kind": "positional or keyword", "annotation": "object | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "GenerationMetadata | None", "docstring": "", "summary": "Function `_generation_metadata` calls `GenerationMetadata`, `getattr`, `bool`; has 2 explicit return paths.", "implementation": "Function `_generation_metadata` calls `GenerationMetadata`, `getattr`, `bool`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 571, "end_line": 583, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L571-L583", "decorators": [], "calls": [ "GenerationMetadata", "getattr", "bool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "GenerationMetadata(no_final_content_watchdog_tokens=getattr(thinking_processor, '_no_final_content_token_limit', None),…" ], "awaits": false, "yields": false }, { "name": "_ThinkingAwareLogitsProcessor", "qualname": "_ThinkingAwareLogitsProcessor", "full_name": "vllm_mlx.server._ThinkingAwareLogitsProcessor", "kind": "class", "signature": "class _ThinkingAwareLogitsProcessor", "parameters": [ { "name": "inner", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_has_think_tag", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "_ThinkingAwareLogitsProcessor", "docstring": "Wrap a ``JSONSchemaLogitsProcessor`` so JSON constraining only activates\nafter the model emits ````, letting it reason freely first.\n\nWithout this wrapper ``enable_thinking`` is forced to ``False`` when\nconstrained decoding is active, which degrades output for thinking models\n(Qwen 3.5/3.6, DeepSeek-R1, etc.) — the model produces degenerated\nwhitespace/brace loops instead of valid JSON because it was trained to\nthink before answering.", "summary": "Wrap a ``JSONSchemaLogitsProcessor`` so JSON constraining only activates after the model emits ````, letting it reason freely first.", "implementation": "Class `_ThinkingAwareLogitsProcessor` declares 5 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 586, "end_line": 697, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L586-L697", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_ThinkingAwareLogitsProcessor.__init__", "full_name": "vllm_mlx.server._ThinkingAwareLogitsProcessor.__init__", "kind": "method", "signature": "def __init__(self, inner, prompt_has_think_tag: bool=False)", "parameters": [ { "name": "inner", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_has_think_tag", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_ThinkingAwareLogitsProcessor.__init__` updates `self._inner`, `self._active`, `self._in_thinking`, `self._waiting_for_json`.", "implementation": "Method `_ThinkingAwareLogitsProcessor.__init__` updates `self._inner`, `self._active`, `self._in_thinking`, `self._waiting_for_json`.", "documented": false, "public": false, "addressable": true, "line": 597, "end_line": 608, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L597-L608", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._inner", "self._active", "self._in_thinking", "self._waiting_for_json", "self._base_prompt_len", "self._json_scan_offset", "self._tokenizer" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_scan_for_json_start", "qualname": "_ThinkingAwareLogitsProcessor._scan_for_json_start", "full_name": "vllm_mlx.server._ThinkingAwareLogitsProcessor._scan_for_json_start", "kind": "method", "signature": "def _scan_for_json_start(self, tokens_list, tokens, logits)", "parameters": [ { "name": "tokens_list", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Scan generated tokens for the first ``{`` or ``[``.\n\nScans from ``_json_scan_offset`` (set when entering the waiting\nphase) so that thinking-span tokens are never considered. After\n50 tokens past the scan offset without a JSON start character the\nenforcer is force-activated as a safety net.", "summary": "Scan generated tokens for the first ``{`` or ``[``.", "implementation": "Method `_ThinkingAwareLogitsProcessor._scan_for_json_start` updates `self._active`, `self._inner._prompt_len`; calls `len`, `range`, `self._tokenizer.decode`, `any`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 610, "end_line": 635, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L610-L635", "decorators": [], "calls": [ "len", "range", "self._tokenizer.decode", "any", "self._inner" ], "state_reads": [ "self._json_scan_offset", "self._tokenizer.decode", "self._tokenizer", "self._inner" ], "state_writes": [ "self._active", "self._inner._prompt_len" ], "raises": [], "return_expressions": [ "self._inner(tokens, logits)", "logits" ], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "_ThinkingAwareLogitsProcessor.__call__", "full_name": "vllm_mlx.server._ThinkingAwareLogitsProcessor.__call__", "kind": "method", "signature": "def __call__(self, tokens, logits)", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_ThinkingAwareLogitsProcessor.__call__` updates `self._base_prompt_len`, `self._in_thinking`, `self._waiting_for_json`, `self._json_scan_offset`; calls `self._inner`, `hasattr`, `tokens.tolist`, `list`; has 3 explicit return paths.", "implementation": "Method `_ThinkingAwareLogitsProcessor.__call__` updates `self._base_prompt_len`, `self._in_thinking`, `self._waiting_for_json`, `self._json_scan_offset`; calls `self._inner`, `hasattr`, `tokens.tolist`, `list`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 637, "end_line": 688, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L637-L688", "decorators": [], "calls": [ "self._inner", "hasattr", "tokens.tolist", "list", "isinstance", "len", "max", "self._scan_for_json_start", "self._tokenizer.decode", "min" ], "state_reads": [ "self._active", "self._inner", "self._base_prompt_len", "self._waiting_for_json", "self._scan_for_json_start", "self._in_thinking", "self._tokenizer.decode", "self._tokenizer" ], "state_writes": [ "self._base_prompt_len", "self._in_thinking", "self._waiting_for_json", "self._json_scan_offset" ], "raises": [], "return_expressions": [ "self._inner(tokens, logits)", "self._scan_for_json_start(tokens_list, tokens, logits)", "logits" ], "awaits": false, "yields": false }, { "name": "schema", "qualname": "_ThinkingAwareLogitsProcessor.schema", "full_name": "vllm_mlx.server._ThinkingAwareLogitsProcessor.schema", "kind": "method", "signature": "def schema(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_ThinkingAwareLogitsProcessor.schema` returns `self._inner.schema`.", "implementation": "Method `_ThinkingAwareLogitsProcessor.schema` returns `self._inner.schema`.", "documented": false, "public": false, "addressable": true, "line": 692, "end_line": 693, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L692-L693", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._inner.schema", "self._inner" ], "state_writes": [], "raises": [], "return_expressions": [ "self._inner.schema" ], "awaits": false, "yields": false }, { "name": "_disabled", "qualname": "_ThinkingAwareLogitsProcessor._disabled", "full_name": "vllm_mlx.server._ThinkingAwareLogitsProcessor._disabled", "kind": "method", "signature": "def _disabled(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_ThinkingAwareLogitsProcessor._disabled` returns `self._inner._disabled`.", "implementation": "Method `_ThinkingAwareLogitsProcessor._disabled` returns `self._inner._disabled`.", "documented": false, "public": false, "addressable": true, "line": 696, "end_line": 697, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L696-L697", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._inner._disabled", "self._inner" ], "state_writes": [], "raises": [], "return_expressions": [ "self._inner._disabled" ], "awaits": false, "yields": false }, { "name": "_attach_response_format_logits_processor", "qualname": "_attach_response_format_logits_processor", "full_name": "vllm_mlx.server._attach_response_format_logits_processor", "kind": "function", "signature": "def _attach_response_format_logits_processor(chat_kwargs: dict, json_logits_processor: object) -> object", "parameters": [ { "name": "chat_kwargs", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "json_logits_processor", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "object", "docstring": "Attach response_format constraints and keep thinking disabled.\n\nresponse_format content must be constrained from the first generated token.\nIf the processor is hidden behind thinking-state handling, direct JSON\nemissions can bypass the constraint and run until max_tokens.", "summary": "Attach response_format constraints and keep thinking disabled.", "implementation": "Function `_attach_response_format_logits_processor` calls `dict`, `chat_kwargs.get`, `list`; returns `json_logits_processor`.", "documented": true, "public": false, "addressable": true, "line": 700, "end_line": 717, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L700-L717", "decorators": [], "calls": [ "dict", "chat_kwargs.get", "list" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json_logits_processor" ], "awaits": false, "yields": false }, { "name": "_coerce_logit_bias", "qualname": "_coerce_logit_bias", "full_name": "vllm_mlx.server._coerce_logit_bias", "kind": "function", "signature": "def _coerce_logit_bias(logit_bias: dict[str, float]) -> dict[int, float]", "parameters": [ { "name": "logit_bias", "kind": "positional or keyword", "annotation": "dict[str, float]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[int, float]", "docstring": "", "summary": "Function `_coerce_logit_bias` calls `logit_bias.items`, `int`, `float`, `HTTPException`; can raise `HTTPException`; returns `coerced`.", "implementation": "Function `_coerce_logit_bias` calls `logit_bias.items`, `int`, `float`, `HTTPException`; can raise `HTTPException`; returns `coerced`.", "documented": false, "public": false, "addressable": true, "line": 720, "end_line": 730, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L720-L730", "decorators": [], "calls": [ "logit_bias.items", "int", "float", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "coerced" ], "awaits": false, "yields": false }, { "name": "_attach_logit_bias_processor", "qualname": "_attach_logit_bias_processor", "full_name": "vllm_mlx.server._attach_logit_bias_processor", "kind": "function", "signature": "def _attach_logit_bias_processor(chat_kwargs: dict, logit_bias: dict[str, float] | None)", "parameters": [ { "name": "chat_kwargs", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logit_bias", "kind": "positional or keyword", "annotation": "dict[str, float] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Function `_attach_logit_bias_processor` calls `make_logits_processors`, `_coerce_logit_bias`, `chat_kwargs.get`, `list`; returns `None`.", "implementation": "Function `_attach_logit_bias_processor` calls `make_logits_processors`, `_coerce_logit_bias`, `chat_kwargs.get`, `list`; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 733, "end_line": 744, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L733-L744", "decorators": [], "calls": [ "make_logits_processors", "_coerce_logit_bias", "chat_kwargs.get", "list" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_prepare_chat_completion_invocation", "qualname": "_prepare_chat_completion_invocation", "full_name": "vllm_mlx.server._prepare_chat_completion_invocation", "kind": "function", "signature": "def _prepare_chat_completion_invocation(engine: BaseEngine, request: ChatCompletionRequest, effective_max_tokens: int) -> PreparedChatInvocation", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "effective_max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "PreparedChatInvocation", "docstring": "Precompute messages, kwargs, and decoding constraints for chat completions.", "summary": "Precompute messages, kwargs, and decoding constraints for chat completions.", "implementation": "Function `_prepare_chat_completion_invocation` calls `_prepare_chat_messages`, `_prepare_json_logits_processor`, `bool`, `_resolve_temperature`; returns `PreparedChatInvocation(messages=messages, chat_kwargs=chat_kwargs, response_format=response_format, json_logits_process…`.", "documented": true, "public": false, "addressable": true, "line": 747, "end_line": 855, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L747-L855", "decorators": [], "calls": [ "_prepare_chat_messages", "_prepare_json_logits_processor", "bool", "_resolve_temperature", "_resolve_top_p", "_resolve_top_k", "_resolve_min_p", "_resolve_presence_penalty", "_resolve_repetition_penalty", "_attach_logit_bias_processor", "getattr", "_resolve_chat_template_kwargs", "convert_tools_for_template", "_apply_forced_tool_choice", "get_parser_stop_tokens", "_attach_response_format_logits_processor", "chat_kwargs.get", "_build_thinking_processor", "list", "PreparedChatInvocation" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "PreparedChatInvocation(messages=messages, chat_kwargs=chat_kwargs, response_format=response_format, json_logits_process…" ], "awaits": false, "yields": false }, { "name": "_prepare_anthropic_invocation", "qualname": "_prepare_anthropic_invocation", "full_name": "vllm_mlx.server._prepare_anthropic_invocation", "kind": "function", "signature": "def _prepare_anthropic_invocation(engine: BaseEngine, openai_request: ChatCompletionRequest, effective_max_tokens: int) -> PreparedChatInvocation", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "openai_request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "effective_max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "PreparedChatInvocation", "docstring": "Precompute messages, kwargs, and decoding constraints for Anthropic API.", "summary": "Precompute messages, kwargs, and decoding constraints for Anthropic API.", "implementation": "Function `_prepare_anthropic_invocation` calls `_prepare_chat_messages`, `_prepare_json_logits_processor`, `bool`, `_resolve_temperature`; returns `PreparedChatInvocation(messages=messages, chat_kwargs=chat_kwargs, response_format=response_format, json_logits_process…`.", "documented": true, "public": false, "addressable": true, "line": 858, "end_line": 910, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L858-L910", "decorators": [], "calls": [ "_prepare_chat_messages", "_prepare_json_logits_processor", "bool", "_resolve_temperature", "_resolve_top_p", "_resolve_top_k", "_resolve_min_p", "_resolve_presence_penalty", "_resolve_repetition_penalty", "_resolve_chat_template_kwargs", "convert_tools_for_template", "_apply_forced_tool_choice", "_attach_response_format_logits_processor", "PreparedChatInvocation" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "PreparedChatInvocation(messages=messages, chat_kwargs=chat_kwargs, response_format=response_format, json_logits_process…" ], "awaits": false, "yields": false }, { "name": "_thinking_disabled", "qualname": "_thinking_disabled", "full_name": "vllm_mlx.server._thinking_disabled", "kind": "function", "signature": "def _thinking_disabled(request, chat_kwargs: dict | None=None) -> bool", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "chat_kwargs", "kind": "positional or keyword", "annotation": "dict | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "bool", "docstring": "Return True iff thinking is explicitly disabled for this request.\n\nChecks both the request-level ``enable_thinking`` field and the resolved\n``chat_template_kwargs`` (which may carry the server-wide default set via\n``--default-chat-template-kwargs``). When thinking is disabled the prompt\ncontains no injected ```` block, so the streaming reasoning parser\nmust not default to implicit-thinking mode and swallow plain content into\na ``thinking`` block.", "summary": "Return True iff thinking is explicitly disabled for this request.", "implementation": "Function `_thinking_disabled` calls `getattr`, `chat_kwargs.get`, `ctk.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 934, "end_line": 950, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L934-L950", "decorators": [], "calls": [ "getattr", "chat_kwargs.get", "ctk.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "_strip_backslash_before_unicode", "qualname": "_strip_backslash_before_unicode", "full_name": "vllm_mlx.server._strip_backslash_before_unicode", "kind": "function", "signature": "def _strip_backslash_before_unicode(obj: object) -> object", "parameters": [ { "name": "obj", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "object", "docstring": "Remove spurious backslashes before non-ASCII chars in JSON string values.\n\nlm-format-enforcer's grammar allows ``\\`` (valid JSON escape) followed by\nnon-ASCII characters such as Korean syllables. The model therefore generates\n``\\빠\\르\\게`` — valid JSON whose decoded value contains literal backslashes.\nThis helper strips those spurious backslashes so clients receive clean text.", "summary": "Remove spurious backslashes before non-ASCII chars in JSON string values.", "implementation": "Function `_strip_backslash_before_unicode` calls `isinstance`, `_strip_backslash_before_unicode`, `obj.items`, `re.sub`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 983, "end_line": 997, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L983-L997", "decorators": [], "calls": [ "isinstance", "_strip_backslash_before_unicode", "obj.items", "re.sub" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{k: _strip_backslash_before_unicode(v) for k, v in obj.items()}", "[_strip_backslash_before_unicode(v) for v in obj]", "re.sub('\\\\\\\\([^\\\\x00-\\\\x7F])', '\\\\1', obj)", "obj" ], "awaits": false, "yields": false }, { "name": "_sanitize_log_text", "qualname": "_sanitize_log_text", "full_name": "vllm_mlx.server._sanitize_log_text", "kind": "function", "signature": "def _sanitize_log_text(value: object, limit: int | None=None) -> str", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "limit", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "str", "docstring": "Escape control characters before logging untrusted text.", "summary": "Escape control characters before logging untrusted text.", "implementation": "Function `_sanitize_log_text` calls `str`, `escaped.append`, `ch.isprintable`, `ord`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1000, "end_line": 1022, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1000-L1022", "decorators": [], "calls": [ "str", "escaped.append", "ch.isprintable", "ord", "''.join", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "sanitized[:limit] + '...'", "sanitized" ], "awaits": false, "yields": false }, { "name": "_log_and_raise_internal_error", "qualname": "_log_and_raise_internal_error", "full_name": "vllm_mlx.server._log_and_raise_internal_error", "kind": "function", "signature": "def _log_and_raise_internal_error(log_prefix: str, exc: Exception, detail: str) -> None", "parameters": [ { "name": "log_prefix", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "exc", "kind": "positional or keyword", "annotation": "Exception", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "detail", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Log a sanitized exception string and raise a generic 500 response.", "summary": "Log a sanitized exception string and raise a generic 500 response.", "implementation": "Function `_log_and_raise_internal_error` calls `logger.error`, `_sanitize_log_text`, `HTTPException`; can raise `HTTPException`.", "documented": true, "public": false, "addressable": true, "line": 1025, "end_line": 1028, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1025-L1028", "decorators": [], "calls": [ "logger.error", "_sanitize_log_text", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_raise_engine_busy", "qualname": "_raise_engine_busy", "full_name": "vllm_mlx.server._raise_engine_busy", "kind": "function", "signature": "def _raise_engine_busy(exc: EngineBusy) -> None", "parameters": [ { "name": "exc", "kind": "positional or keyword", "annotation": "EngineBusy", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Translate serialized-engine admission failures into retryable HTTP 503.", "summary": "Translate serialized-engine admission failures into retryable HTTP 503.", "implementation": "Function `_raise_engine_busy` calls `HTTPException`, `str`; can raise `HTTPException`.", "documented": true, "public": false, "addressable": true, "line": 1031, "end_line": 1039, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1031-L1039", "decorators": [], "calls": [ "HTTPException", "str" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "RequestModelContext", "qualname": "RequestModelContext", "full_name": "vllm_mlx.server.RequestModelContext", "kind": "class", "signature": "class RequestModelContext", "parameters": [ { "name": "model_name", "kind": "field", "annotation": "str", "default": "", "required": true, "description": "Required constructor field." }, { "name": "engine", "kind": "field", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required constructor field." }, { "name": "lease", "kind": "field", "annotation": "ModelLease | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "RequestModelContext", "docstring": "Request-scoped engine/lease context.", "summary": "Request-scoped engine/lease context.", "implementation": "Class `RequestModelContext` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 1043, "end_line": 1056, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1043-L1056", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "release", "qualname": "RequestModelContext.release", "full_name": "vllm_mlx.server.RequestModelContext.release", "kind": "method", "signature": "async def release(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Release the registry lease once, if this context owns one.", "summary": "Release the registry lease once, if this context owns one.", "implementation": "Method `RequestModelContext.release` updates `self.lease`; calls `lease.release`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 1050, "end_line": 1056, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1050-L1056", "decorators": [], "calls": [ "lease.release" ], "state_reads": [ "self.lease" ], "state_writes": [ "self.lease" ], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_list_available_model_names", "qualname": "_list_available_model_names", "full_name": "vllm_mlx.server._list_available_model_names", "kind": "function", "signature": "def _list_available_model_names() -> list[str]", "parameters": [], "return_annotation": "list[str]", "docstring": "", "summary": "Function `_list_available_model_names` has 2 explicit return paths.", "implementation": "Function `_list_available_model_names` has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1059, "end_line": 1062, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1059-L1062", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_model_manager.registered_model_names", "[_model_name] if _model_name else []" ], "awaits": false, "yields": false }, { "name": "_response_model_name", "qualname": "_response_model_name", "full_name": "vllm_mlx.server._response_model_name", "kind": "function", "signature": "def _response_model_name(request_model: str) -> str", "parameters": [ { "name": "request_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Return the response model field for single-model or registry mode.", "summary": "Return the response model field for single-model or registry mode.", "implementation": "Function `_response_model_name` returns `_model_name or request_model`.", "documented": true, "public": false, "addressable": true, "line": 1065, "end_line": 1067, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1065-L1067", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_model_name or request_model" ], "awaits": false, "yields": false }, { "name": "_acquire_request_model", "qualname": "_acquire_request_model", "full_name": "vllm_mlx.server._acquire_request_model", "kind": "function", "signature": "async def _acquire_request_model(request_model: str) -> RequestModelContext", "parameters": [ { "name": "request_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "RequestModelContext", "docstring": "Acquire the model/engine that should serve this request.", "summary": "Acquire the model/engine that should serve this request.", "implementation": "Function `_acquire_request_model` calls `_validate_model_name`, `get_engine`, `_detect_native_tool_support`, `_detect_harmony_rendering`; awaits asynchronous work; can raise `HTTPException`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1070, "end_line": 1094, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1070-L1094", "decorators": [], "calls": [ "_validate_model_name", "get_engine", "_detect_native_tool_support", "_detect_harmony_rendering", "RequestModelContext", "_model_manager.acquire", "HTTPException", "str" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "RequestModelContext(model_name=_model_name or request_model, engine=engine)", "RequestModelContext(model_name=request_model, engine=lease.engine, lease=lease)" ], "awaits": true, "yields": false }, { "name": "_stream_with_model_context", "qualname": "_stream_with_model_context", "full_name": "vllm_mlx.server._stream_with_model_context", "kind": "function", "signature": "async def _stream_with_model_context(context: RequestModelContext, stream: AsyncIterator[str]) -> AsyncIterator[str]", "parameters": [ { "name": "context", "kind": "positional or keyword", "annotation": "RequestModelContext", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "stream", "kind": "positional or keyword", "annotation": "AsyncIterator[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "AsyncIterator[str]", "docstring": "Ensure model leases survive for the full streaming response.", "summary": "Ensure model leases survive for the full streaming response.", "implementation": "Function `_stream_with_model_context` calls `context.release`; awaits asynchronous work; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 1097, "end_line": 1106, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1097-L1106", "decorators": [], "calls": [ "context.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": true }, { "name": "_build_tool_parser", "qualname": "_build_tool_parser", "full_name": "vllm_mlx.server._build_tool_parser", "kind": "function", "signature": "def _build_tool_parser(engine: BaseEngine | None)", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Create a fresh tool parser instance for a single request/stream.", "summary": "Create a fresh tool parser instance for a single request/stream.", "implementation": "Function `_build_tool_parser` calls `type`, `ToolParserManager.get_tool_parser`, `_get_engine_tokenizer`, `parser_cls`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1109, "end_line": 1123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1109-L1123", "decorators": [], "calls": [ "type", "ToolParserManager.get_tool_parser", "_get_engine_tokenizer", "parser_cls" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "parser_cls(tokenizer)", "parser_cls()" ], "awaits": false, "yields": false }, { "name": "_build_reasoning_parser", "qualname": "_build_reasoning_parser", "full_name": "vllm_mlx.server._build_reasoning_parser", "kind": "function", "signature": "def _build_reasoning_parser(engine: BaseEngine | None=None)", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Create a fresh reasoning parser instance for a single request/stream.", "summary": "Create a fresh reasoning parser instance for a single request/stream.", "implementation": "Function `_build_reasoning_parser` calls `getattr`, `get_reasoning_parser`, `parser_cls`, `type(_reasoning_parser)`; has 5 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1126, "end_line": 1140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1126-L1140", "decorators": [], "calls": [ "getattr", "get_reasoning_parser", "parser_cls", "type(_reasoning_parser)", "type" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parser_cls(tokenizer)", "parser_cls()", "None", "type(_reasoning_parser)(tokenizer)", "type(_reasoning_parser)()" ], "awaits": false, "yields": false }, { "name": "_prepare_streaming_reasoning_parser", "qualname": "_prepare_streaming_reasoning_parser", "full_name": "vllm_mlx.server._prepare_streaming_reasoning_parser", "kind": "function", "signature": "def _prepare_streaming_reasoning_parser(engine: BaseEngine, request: ChatCompletionRequest | ResponsesRequest | None, chat_kwargs: dict[str, object], *, allowed: bool=True)", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest | ResponsesRequest | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "chat_kwargs", "kind": "positional or keyword", "annotation": "dict[str, object]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "allowed", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "not annotated", "docstring": "Build and reset request-local reasoning state when thinking is enabled.", "summary": "Build and reset request-local reasoning state when thinking is enabled.", "implementation": "Function `_prepare_streaming_reasoning_parser` calls `_thinking_disabled`, `_build_reasoning_parser`, `parser.reset_state`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1143, "end_line": 1156, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1143-L1156", "decorators": [], "calls": [ "_thinking_disabled", "_build_reasoning_parser", "parser.reset_state" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "parser" ], "awaits": false, "yields": false }, { "name": "_prepare_openai_stream_reasoning_state", "qualname": "_prepare_openai_stream_reasoning_state", "full_name": "vllm_mlx.server._prepare_openai_stream_reasoning_state", "kind": "function", "signature": "def _prepare_openai_stream_reasoning_state(engine: BaseEngine, request: ChatCompletionRequest, chat_kwargs: dict[str, object]) -> tuple[object | None, bool]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "chat_kwargs", "kind": "positional or keyword", "annotation": "dict[str, object]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[object | None, bool]", "docstring": "Return request-local reasoning state and the legacy Nemotron marker state.", "summary": "Return request-local reasoning state and the legacy Nemotron marker state.", "implementation": "Function `_prepare_openai_stream_reasoning_state` calls `_prepare_streaming_reasoning_parser`, `(engine.model_name or '').lower`, `_thinking_disabled`; returns `(parser, is_thinking_model)`.", "documented": true, "public": false, "addressable": true, "line": 1159, "end_line": 1171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1159-L1171", "decorators": [], "calls": [ "_prepare_streaming_reasoning_parser", "(engine.model_name or '').lower", "_thinking_disabled" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(parser, is_thinking_model)" ], "awaits": false, "yields": false }, { "name": "_request_tool_definitions", "qualname": "_request_tool_definitions", "full_name": "vllm_mlx.server._request_tool_definitions", "kind": "function", "signature": "def _request_tool_definitions(request: ChatCompletionRequest) -> list | None", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list | None", "docstring": "Return the request tool schema once for streaming argument coercion.", "summary": "Return the request tool schema once for streaming argument coercion.", "implementation": "Function `_request_tool_definitions` calls `request.model_dump(include={'tools'}).get`, `request.model_dump`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1174, "end_line": 1178, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1174-L1178", "decorators": [], "calls": [ "request.model_dump(include={'tools'}).get", "request.model_dump" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "request.model_dump(include={'tools'}).get('tools')", "None" ], "awaits": false, "yields": false }, { "name": "_streaming_json_fence_stripper", "qualname": "_streaming_json_fence_stripper", "full_name": "vllm_mlx.server._streaming_json_fence_stripper", "kind": "function", "signature": "def _streaming_json_fence_stripper(request: ChatCompletionRequest) -> StreamingJsonFenceStripper | None", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "StreamingJsonFenceStripper | None", "docstring": "Create a fence stripper only for JSON-constrained streaming responses.", "summary": "Create a fence stripper only for JSON-constrained streaming responses.", "implementation": "Function `_streaming_json_fence_stripper` calls `getattr`, `isinstance`, `response_format.get`, `StreamingJsonFenceStripper`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1181, "end_line": 1191, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1181-L1191", "decorators": [], "calls": [ "getattr", "isinstance", "response_format.get", "StreamingJsonFenceStripper" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "StreamingJsonFenceStripper()", "None" ], "awaits": false, "yields": false }, { "name": "_get_idle_unload_event", "qualname": "_get_idle_unload_event", "full_name": "vllm_mlx.server._get_idle_unload_event", "kind": "function", "signature": "def _get_idle_unload_event() -> asyncio.Event", "parameters": [], "return_annotation": "asyncio.Event", "docstring": "Return the idle-unload gate event, creating it on first use.\n\nThe returned Event is bound to the running loop at creation time.\nReset ``_idle_unload_enabled`` to ``None`` when tearing down the\nserver or switching event loops (e.g. in test fixtures).", "summary": "Return the idle-unload gate event, creating it on first use.", "implementation": "Function `_get_idle_unload_event` calls `asyncio.Event`, `_idle_unload_enabled.set`; returns `_idle_unload_enabled`.", "documented": true, "public": false, "addressable": true, "line": 1206, "end_line": 1217, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1206-L1217", "decorators": [], "calls": [ "asyncio.Event", "_idle_unload_enabled.set" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_idle_unload_enabled" ], "awaits": false, "yields": false }, { "name": "_invalidate_tool_parser_cache", "qualname": "_invalidate_tool_parser_cache", "full_name": "vllm_mlx.server._invalidate_tool_parser_cache", "kind": "function", "signature": "def _invalidate_tool_parser_cache(reason: str | None=None) -> None", "parameters": [ { "name": "reason", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Drop cached parser state when the serving tokenizer changes.", "summary": "Drop cached parser state when the serving tokenizer changes.", "implementation": "Function `_invalidate_tool_parser_cache` calls `logger.debug`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1220, "end_line": 1229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1220-L1229", "decorators": [], "calls": [ "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_load_prefix_cache_from_disk", "qualname": "_load_prefix_cache_from_disk", "full_name": "vllm_mlx.server._load_prefix_cache_from_disk", "kind": "function", "signature": "def _load_prefix_cache_from_disk(engine: BaseEngine | None=None) -> None", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Load prefix cache from disk during startup.", "summary": "Load prefix cache from disk during startup.", "implementation": "Function `_load_prefix_cache_from_disk` calls `_get_cache_dir`, `logger.info`, `target_engine.load_cache_from_disk`, `logger.warning`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1232, "end_line": 1250, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1232-L1250", "decorators": [], "calls": [ "_get_cache_dir", "logger.info", "target_engine.load_cache_from_disk", "logger.warning", "_sanitize_log_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_save_prefix_cache_to_disk", "qualname": "_save_prefix_cache_to_disk", "full_name": "vllm_mlx.server._save_prefix_cache_to_disk", "kind": "function", "signature": "def _save_prefix_cache_to_disk(engine: BaseEngine | None=None) -> None", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Save prefix cache to disk during shutdown.", "summary": "Save prefix cache to disk during shutdown.", "implementation": "Function `_save_prefix_cache_to_disk` calls `_get_cache_dir`, `logger.info`, `target_engine.save_cache_to_disk`, `logger.warning`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1253, "end_line": 1271, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1253-L1271", "decorators": [], "calls": [ "_get_cache_dir", "logger.info", "target_engine.save_cache_to_disk", "logger.warning", "_sanitize_log_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_get_cache_dir", "qualname": "_get_cache_dir", "full_name": "vllm_mlx.server._get_cache_dir", "kind": "function", "signature": "def _get_cache_dir() -> str", "parameters": [], "return_annotation": "str", "docstring": "Get cache persistence directory based on actual model path.", "summary": "Get cache persistence directory based on actual model path.", "implementation": "Function `_get_cache_dir` calls `logger.info`, `type`, `str(model_name).replace('/', '--').replace`, `str(model_name).replace`; returns `cache_dir`.", "documented": true, "public": false, "addressable": true, "line": 1274, "end_line": 1290, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1274-L1290", "decorators": [], "calls": [ "logger.info", "type", "str(model_name).replace('/', '--').replace", "str(model_name).replace", "str", "os.path.join", "os.path.expanduser" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cache_dir" ], "awaits": false, "yields": false }, { "name": "_build_engine", "qualname": "_build_engine", "full_name": "vllm_mlx.server._build_engine", "kind": "function", "signature": "def _build_engine(spec: ModelSpec) -> BaseEngine", "parameters": [ { "name": "spec", "kind": "positional or keyword", "annotation": "ModelSpec", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BaseEngine", "docstring": "Construct an engine instance from a model spec without starting it.", "summary": "Construct an engine instance from a model spec without starting it.", "implementation": "Function `_build_engine` calls `logger.info`, `BatchedEngine`, `getattr`, `SimpleEngine`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1293, "end_line": 1323, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1293-L1323", "decorators": [], "calls": [ "logger.info", "BatchedEngine", "getattr", "SimpleEngine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "BatchedEngine(model_name=spec.model_name, scheduler_config=spec.scheduler_config, stream_interval=spec.stream_interval,…", "SimpleEngine(model_name=spec.model_name, force_mllm=spec.force_mllm, mtp=spec.mtp, prefill_step_size=spec.prefill_step_…" ], "awaits": false, "yields": false }, { "name": "_engine_factory", "qualname": "_engine_factory", "full_name": "vllm_mlx.server._engine_factory", "kind": "function", "signature": "async def _engine_factory(spec: ModelSpec) -> BaseEngine", "parameters": [ { "name": "spec", "kind": "positional or keyword", "annotation": "ModelSpec", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BaseEngine", "docstring": "Async engine factory used by the residency manager.", "summary": "Async engine factory used by the residency manager.", "implementation": "Function `_engine_factory` calls `_build_engine`; returns `_build_engine(spec)`.", "documented": true, "public": false, "addressable": true, "line": 1326, "end_line": 1328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1326-L1328", "decorators": [], "calls": [ "_build_engine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_build_engine(spec)" ], "awaits": false, "yields": false }, { "name": "_run_blocking_engine_cache_io", "qualname": "_run_blocking_engine_cache_io", "full_name": "vllm_mlx.server._run_blocking_engine_cache_io", "kind": "function", "signature": "async def _run_blocking_engine_cache_io(io_fn, engine: BaseEngine) -> None", "parameters": [ { "name": "io_fn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Run blocking cache persistence off the event loop.\n\nIf the caller is canceled while waiting, finish the in-flight thread before\npropagating cancellation so engine state cannot keep mutating in the\nbackground after lifecycle cleanup has started.", "summary": "Run blocking cache persistence off the event loop.", "implementation": "Function `_run_blocking_engine_cache_io` calls `asyncio.create_task`, `asyncio.to_thread`, `asyncio.shield`, `suspend_cancellation`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 1331, "end_line": 1350, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1331-L1350", "decorators": [], "calls": [ "asyncio.create_task", "asyncio.to_thread", "asyncio.shield", "suspend_cancellation", "task.done" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_restore_engine_state", "qualname": "_restore_engine_state", "full_name": "vllm_mlx.server._restore_engine_state", "kind": "function", "signature": "async def _restore_engine_state(spec: ModelSpec, engine: BaseEngine) -> None", "parameters": [ { "name": "spec", "kind": "positional or keyword", "annotation": "ModelSpec", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Restore engine-local state, such as prefix cache, after a cold load.", "summary": "Restore engine-local state, such as prefix cache, after a cold load.", "implementation": "Function `_restore_engine_state` calls `hasattr`, `_run_blocking_engine_cache_io`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 1353, "end_line": 1356, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1353-L1356", "decorators": [], "calls": [ "hasattr", "_run_blocking_engine_cache_io" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_persist_engine_state", "qualname": "_persist_engine_state", "full_name": "vllm_mlx.server._persist_engine_state", "kind": "function", "signature": "async def _persist_engine_state(spec: ModelSpec, engine: BaseEngine) -> None", "parameters": [ { "name": "spec", "kind": "positional or keyword", "annotation": "ModelSpec", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Persist engine-local state before an idle unload or shutdown unload.", "summary": "Persist engine-local state before an idle unload or shutdown unload.", "implementation": "Function `_persist_engine_state` calls `hasattr`, `_run_blocking_engine_cache_io`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 1359, "end_line": 1362, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1359-L1362", "decorators": [], "calls": [ "hasattr", "_run_blocking_engine_cache_io" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_activate_engine", "qualname": "_activate_engine", "full_name": "vllm_mlx.server._activate_engine", "kind": "function", "signature": "def _activate_engine(engine: BaseEngine | None) -> BaseEngine | None", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "BaseEngine | None", "docstring": "Set the global engine pointer and refresh parser-sensitive state.", "summary": "Set the global engine pointer and refresh parser-sensitive state.", "implementation": "Function `_activate_engine` calls `_invalidate_tool_parser_cache`, `_detect_native_tool_support`, `_detect_harmony_rendering`; returns `_engine`.", "documented": true, "public": false, "addressable": true, "line": 1365, "end_line": 1375, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1365-L1375", "decorators": [], "calls": [ "_invalidate_tool_parser_cache", "_detect_native_tool_support", "_detect_harmony_rendering" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_engine" ], "awaits": false, "yields": false }, { "name": "_sync_engine_from_residency", "qualname": "_sync_engine_from_residency", "full_name": "vllm_mlx.server._sync_engine_from_residency", "kind": "function", "signature": "def _sync_engine_from_residency() -> BaseEngine | None", "parameters": [], "return_annotation": "BaseEngine | None", "docstring": "Sync the global engine pointer from the residency manager state.\n\nSafety: all callers run on the single-threaded asyncio event loop and do not\nyield between reading the residency state and writing ``_engine``, so no\nadditional locking is required.", "summary": "Sync the global engine pointer from the residency manager state.", "implementation": "Function `_sync_engine_from_residency` calls `_activate_engine`, `_residency_manager.get_engine`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1378, "end_line": 1388, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1378-L1388", "decorators": [], "calls": [ "_activate_engine", "_residency_manager.get_engine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_engine", "_activate_engine(_residency_manager.get_engine(_default_model_key))" ], "awaits": false, "yields": false }, { "name": "_get_lifecycle_status", "qualname": "_get_lifecycle_status", "full_name": "vllm_mlx.server._get_lifecycle_status", "kind": "function", "signature": "def _get_lifecycle_status() -> dict | None", "parameters": [], "return_annotation": "dict | None", "docstring": "Get lifecycle status for the default resident if lifecycle is enabled.", "summary": "Get lifecycle status for the default resident if lifecycle is enabled.", "implementation": "Function `_get_lifecycle_status` calls `_residency_manager.get_status`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1391, "end_line": 1395, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1391-L1395", "decorators": [], "calls": [ "_residency_manager.get_status" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "_residency_manager.get_status(_default_model_key)" ], "awaits": false, "yields": false }, { "name": "_public_lifecycle_status", "qualname": "_public_lifecycle_status", "full_name": "vllm_mlx.server._public_lifecycle_status", "kind": "function", "signature": "def _public_lifecycle_status(lifecycle: dict | None) -> dict | None", "parameters": [ { "name": "lifecycle", "kind": "positional or keyword", "annotation": "dict | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict | None", "docstring": "Return residency status safe for unauthenticated public endpoints.", "summary": "Return residency status safe for unauthenticated public endpoints.", "implementation": "Function `_public_lifecycle_status` calls `dict`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1398, "end_line": 1410, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1398-L1410", "decorators": [], "calls": [ "dict" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "public" ], "awaits": false, "yields": false }, { "name": "_lifecycle_loop", "qualname": "_lifecycle_loop", "full_name": "vllm_mlx.server._lifecycle_loop", "kind": "function", "signature": "async def _lifecycle_loop() -> None", "parameters": [], "return_annotation": "None", "docstring": "Background idle-unload loop for the default resident.", "summary": "Background idle-unload loop for the default resident.", "implementation": "Function `_lifecycle_loop` calls `asyncio.sleep`, `_get_idle_unload_event().wait`, `_get_idle_unload_event`, `_residency_manager.unload_if_idle`; awaits asynchronous work.", "documented": true, "public": false, "addressable": true, "line": 1413, "end_line": 1433, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1413-L1433", "decorators": [], "calls": [ "asyncio.sleep", "_get_idle_unload_event().wait", "_get_idle_unload_event", "_residency_manager.unload_if_idle", "logger.exception", "_sync_engine_from_residency", "min" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_acquire_default_engine", "qualname": "_acquire_default_engine", "full_name": "vllm_mlx.server._acquire_default_engine", "kind": "function", "signature": "async def _acquire_default_engine(*, count_activity: bool=True) -> BaseEngine", "parameters": [ { "name": "count_activity", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "BaseEngine", "docstring": "Acquire the default engine, auto-loading via the residency manager if needed.", "summary": "Acquire the default engine, auto-loading via the residency manager if needed.", "implementation": "Function `_acquire_default_engine` calls `get_engine`, `_residency_manager.acquire`, `_activate_engine`, `HTTPException`; awaits asynchronous work; can raise `HTTPException`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1436, "end_line": 1451, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1436-L1451", "decorators": [], "calls": [ "get_engine", "_residency_manager.acquire", "_activate_engine", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "get_engine()", "activated_engine" ], "awaits": true, "yields": false }, { "name": "_release_default_engine", "qualname": "_release_default_engine", "full_name": "vllm_mlx.server._release_default_engine", "kind": "function", "signature": "async def _release_default_engine(*, count_activity: bool=True) -> None", "parameters": [ { "name": "count_activity", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "None", "docstring": "Release the default engine after request processing.", "summary": "Release the default engine after request processing.", "implementation": "Function `_release_default_engine` calls `_residency_manager.release`, `_sync_engine_from_residency`; awaits asynchronous work; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1454, "end_line": 1463, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1454-L1463", "decorators": [], "calls": [ "_residency_manager.release", "_sync_engine_from_residency" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "lifespan", "qualname": "lifespan", "full_name": "vllm_mlx.server.lifespan", "kind": "function", "signature": "async def lifespan(app: FastAPI)", "parameters": [ { "name": "app", "kind": "positional or keyword", "annotation": "FastAPI", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "FastAPI lifespan for startup/shutdown events.", "summary": "FastAPI lifespan for startup/shutdown events.", "implementation": "Function `lifespan` calls `_get_idle_unload_event().clear`, `_get_idle_unload_event`, `_residency_manager.ensure_loaded`, `_sync_engine_from_residency`; awaits asynchronous work; yields values incrementally; can raise `primary_exc`, `cleanup_exc`.", "documented": true, "public": true, "addressable": true, "line": 1466, "end_line": 1589, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1466-L1589", "decorators": [], "calls": [ "_get_idle_unload_event().clear", "_get_idle_unload_event", "_residency_manager.ensure_loaded", "_sync_engine_from_residency", "hasattr", "_engine.start", "_model_manager.preload", "_load_prefix_cache_from_disk", "load_warmup_file", "logger.info", "len", "warm_prefix_cache", "result.get", "logger.warning", "_sanitize_log_text", "asyncio.create_task", "_lifecycle_loop", "os.environ.get", "init_mcp", "_get_idle_unload_event().set", "_save_prefix_cache_to_disk", "_lifecycle_task.cancel", "suppress", "_mcp_manager.stop", "_residency_manager.shutdown", "_engine.stop", "_model_manager.shutdown", "logger.error", "type" ], "state_reads": [], "state_writes": [], "raises": [ "primary_exc", "cleanup_exc" ], "return_expressions": [], "awaits": true, "yields": true }, { "name": "_metrics_result_from_status", "qualname": "_metrics_result_from_status", "full_name": "vllm_mlx.server._metrics_result_from_status", "kind": "function", "signature": "def _metrics_result_from_status(status_code: int) -> str", "parameters": [ { "name": "status_code", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Map HTTP-ish status codes to low-cardinality inference results.", "summary": "Map HTTP-ish status codes to low-cardinality inference results.", "implementation": "Function `_metrics_result_from_status` has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1602, "end_line": 1610, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1602-L1610", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'client_closed'", "'timeout'", "'error'", "'success'" ], "awaits": false, "yields": false }, { "name": "_metrics_path_for_request", "qualname": "_metrics_path_for_request", "full_name": "vllm_mlx.server._metrics_path_for_request", "kind": "function", "signature": "def _metrics_path_for_request(request: Request) -> str", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Prefer route templates over raw URLs to keep metrics cardinality bounded.", "summary": "Prefer route templates over raw URLs to keep metrics cardinality bounded.", "implementation": "Function `_metrics_path_for_request` calls `request.scope.get`, `getattr`, `str`, `candidate.matches`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1613, "end_line": 1626, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1613-L1626", "decorators": [], "calls": [ "request.scope.get", "getattr", "str", "candidate.matches" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "str(path)", "'__unmatched__'" ], "awaits": false, "yields": false }, { "name": "_metrics_middleware", "qualname": "_metrics_middleware", "full_name": "vllm_mlx.server._metrics_middleware", "kind": "function", "signature": "async def _metrics_middleware(request: Request, call_next)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "call_next", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Capture generic HTTP request metrics when enabled.", "summary": "Capture generic HTTP request metrics when enabled.", "implementation": "Function `_metrics_middleware` calls `call_next`, `_metrics_path_for_request`, `time.perf_counter`, `_metrics.observe_http_start`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1630, "end_line": 1659, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1630-L1659", "decorators": [ "app.middleware('http')" ], "calls": [ "call_next", "_metrics_path_for_request", "time.perf_counter", "_metrics.observe_http_start", "_metrics.observe_http_finish" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "await call_next(request)", "response" ], "awaits": true, "yields": false }, { "name": "RateLimiter", "qualname": "RateLimiter", "full_name": "vllm_mlx.server.RateLimiter", "kind": "class", "signature": "class RateLimiter", "parameters": [ { "name": "requests_per_minute", "kind": "positional or keyword", "annotation": "int", "default": "60", "required": false, "description": "Optional positional or keyword input; defaults to `60`." }, { "name": "enabled", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "RateLimiter", "docstring": "Simple in-memory rate limiter using sliding window.", "summary": "Simple in-memory rate limiter using sliding window.", "implementation": "Class `RateLimiter` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 1662, "end_line": 1700, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1662-L1700", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "RateLimiter.__init__", "full_name": "vllm_mlx.server.RateLimiter.__init__", "kind": "method", "signature": "def __init__(self, requests_per_minute: int=60, enabled: bool=False)", "parameters": [ { "name": "requests_per_minute", "kind": "positional or keyword", "annotation": "int", "default": "60", "required": false, "description": "Optional positional or keyword input; defaults to `60`." }, { "name": "enabled", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Optional positional or keyword input; defaults to `False`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `RateLimiter.__init__` updates `self.requests_per_minute`, `self.enabled`, `self.window_size`, `self._requests`; calls `defaultdict`, `threading.Lock`.", "implementation": "Method `RateLimiter.__init__` updates `self.requests_per_minute`, `self.enabled`, `self.window_size`, `self._requests`; calls `defaultdict`, `threading.Lock`.", "documented": false, "public": false, "addressable": true, "line": 1665, "end_line": 1670, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1665-L1670", "decorators": [], "calls": [ "defaultdict", "threading.Lock" ], "state_reads": [], "state_writes": [ "self.requests_per_minute", "self.enabled", "self.window_size", "self._requests", "self._lock" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_allowed", "qualname": "RateLimiter.is_allowed", "full_name": "vllm_mlx.server.RateLimiter.is_allowed", "kind": "method", "signature": "def is_allowed(self, client_id: str) -> tuple[bool, int]", "parameters": [ { "name": "client_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, int]", "docstring": "Check if request is allowed for client.\n\nReturns:\n (is_allowed, retry_after_seconds)", "summary": "Check if request is allowed for client.", "implementation": "Method `RateLimiter.is_allowed` calls `time.time`, `len`, `min`, `int`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1672, "end_line": 1700, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1672-L1700", "decorators": [], "calls": [ "time.time", "len", "min", "int", "max", "self._requests[client_id].append" ], "state_reads": [ "self.enabled", "self.window_size", "self._lock", "self._requests", "self.requests_per_minute" ], "state_writes": [], "raises": [], "return_expressions": [ "(True, 0)", "(False, max(1, retry_after))" ], "awaits": false, "yields": false }, { "name": "check_rate_limit", "qualname": "check_rate_limit", "full_name": "vllm_mlx.server.check_rate_limit", "kind": "function", "signature": "async def check_rate_limit(request: Request)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Rate limiting dependency.", "summary": "Rate limiting dependency.", "implementation": "Function `check_rate_limit` calls `request.headers.get`, `_rate_limiter.is_allowed`, `HTTPException`, `str`; can raise `HTTPException`.", "documented": true, "public": true, "addressable": true, "line": 1707, "end_line": 1720, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1707-L1720", "decorators": [], "calls": [ "request.headers.get", "_rate_limiter.is_allowed", "HTTPException", "str" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "verify_api_key", "qualname": "verify_api_key", "full_name": "vllm_mlx.server.verify_api_key", "kind": "function", "signature": "async def verify_api_key(credentials: HTTPAuthorizationCredentials=Depends(security))", "parameters": [ { "name": "credentials", "kind": "positional or keyword", "annotation": "HTTPAuthorizationCredentials", "default": "Depends(security)", "required": false, "description": "Optional positional or keyword input; defaults to `Depends(security)`." } ], "return_annotation": "not annotated", "docstring": "Verify API key if authentication is enabled.", "summary": "Verify API key if authentication is enabled.", "implementation": "Function `verify_api_key` calls `logger.warning`, `HTTPException`, `secrets.compare_digest`; can raise `HTTPException`; returns `True`.", "documented": true, "public": true, "addressable": true, "line": 1723, "end_line": 1742, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1723-L1742", "decorators": [], "calls": [ "logger.warning", "HTTPException", "secrets.compare_digest" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "True" ], "awaits": false, "yields": false }, { "name": "get_engine", "qualname": "get_engine", "full_name": "vllm_mlx.server.get_engine", "kind": "function", "signature": "def get_engine() -> BaseEngine", "parameters": [], "return_annotation": "BaseEngine", "docstring": "Get the loaded engine, raising error if not loaded.", "summary": "Get the loaded engine, raising error if not loaded.", "implementation": "Function `get_engine` calls `HTTPException`; can raise `HTTPException`; returns `_engine`.", "documented": true, "public": true, "addressable": true, "line": 1745, "end_line": 1749, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1745-L1749", "decorators": [], "calls": [ "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "_engine" ], "awaits": false, "yields": false }, { "name": "_coerce_tool_arguments", "qualname": "_coerce_tool_arguments", "full_name": "vllm_mlx.server._coerce_tool_arguments", "kind": "function", "signature": "def _coerce_tool_arguments(arguments_json: str, tool_name: str, tools: list[dict] | None) -> str", "parameters": [ { "name": "arguments_json", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Coerce tool call arguments to match the tool schema.\n\nIf a schema field expects \"string\" but the model produced an object/array,\nJSON-stringify the value. This fixes a common LLM failure mode where models\noutput raw JSON objects instead of JSON strings for file content, etc.", "summary": "Coerce tool call arguments to match the tool schema.", "implementation": "Function `_coerce_tool_arguments` calls `isinstance`, `tool.get('function', {}).get`, `tool.get`, `tool['function'].get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1752, "end_line": 1796, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1752-L1796", "decorators": [], "calls": [ "isinstance", "tool.get('function', {}).get", "tool.get", "tool['function'].get", "json.loads", "schema.get", "arguments.items", "properties[key].get", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "arguments_json", "json.dumps(arguments, ensure_ascii=False)" ], "awaits": false, "yields": false }, { "name": "_validate_model_name", "qualname": "_validate_model_name", "full_name": "vllm_mlx.server._validate_model_name", "kind": "function", "signature": "def _validate_model_name(request_model: str) -> None", "parameters": [ { "name": "request_model", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Validate that the request model name matches the served model.", "summary": "Validate that the request model name matches the served model.", "implementation": "Function `_validate_model_name` calls `_model_manager.has_model`, `', '.join`, `_list_available_model_names`, `HTTPException`; can raise `HTTPException`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1799, "end_line": 1818, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1799-L1818", "decorators": [], "calls": [ "_model_manager.has_model", "', '.join", "_list_available_model_names", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_get_engine_tokenizer", "qualname": "_get_engine_tokenizer", "full_name": "vllm_mlx.server._get_engine_tokenizer", "kind": "function", "signature": "def _get_engine_tokenizer(engine: BaseEngine | None) -> object | None", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "object | None", "docstring": "Return tokenizer-like parser state from the active engine.", "summary": "Return tokenizer-like parser state from the active engine.", "implementation": "Function `_get_engine_tokenizer` calls `getattr`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1821, "end_line": 1828, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1821-L1828", "decorators": [], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "tokenizer", "getattr(engine, '_tokenizer', None)" ], "awaits": false, "yields": false }, { "name": "_get_or_init_tool_parser", "qualname": "_get_or_init_tool_parser", "full_name": "vllm_mlx.server._get_or_init_tool_parser", "kind": "function", "signature": "def _get_or_init_tool_parser(engine: BaseEngine | None=None)", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Return the cached tool parser, initializing it from the given engine.", "summary": "Return the cached tool parser, initializing it from the given engine.", "implementation": "Function `_get_or_init_tool_parser` calls `ToolParserManager.get_tool_parser`, `_get_engine_tokenizer`, `parser_cls`, `logger.info`; returns `_tool_parser_instance`.", "documented": true, "public": false, "addressable": true, "line": 1831, "end_line": 1841, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1831-L1841", "decorators": [], "calls": [ "ToolParserManager.get_tool_parser", "_get_engine_tokenizer", "parser_cls", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_tool_parser_instance" ], "awaits": false, "yields": false }, { "name": "_parse_tool_calls_with_parser", "qualname": "_parse_tool_calls_with_parser", "full_name": "vllm_mlx.server._parse_tool_calls_with_parser", "kind": "function", "signature": "def _parse_tool_calls_with_parser(output_text: str, request: ChatCompletionRequest | None=None, engine: BaseEngine | None=None) -> tuple[str, list | None]", "parameters": [ { "name": "output_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The model output text" }, { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest | None", "default": "None", "required": false, "description": "The original request (for context)" }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "The request-local engine to use for parser initialization" } ], "return_annotation": "tuple[str, list | None]", "docstring": "Parse tool calls from model output using the configured parser.\n\nIf --enable-auto-tool-choice is set with --tool-call-parser, uses the\nselected parser. Otherwise falls back to the generic parse_tool_calls.\n\nArgs:\n output_text: The model output text\n request: The original request (for context)\n engine: The request-local engine to use for parser initialization\n\nReturns:\n Tuple of (cleaned_text, tool_calls)", "summary": "Parse tool calls from model output using the configured parser.", "implementation": "Function `_parse_tool_calls_with_parser` calls `request.model_dump`, `getattr`, `request_dict.get`, `parse_tool_calls`; has 6 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1844, "end_line": 1930, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1844-L1930", "decorators": [], "calls": [ "request.model_dump", "getattr", "request_dict.get", "parse_tool_calls", "_get_or_init_tool_parser", "logger.warning", "_sanitize_log_text", "_tool_parser_instance.reset", "_tool_parser_instance.extract_tool_calls", "ToolCall", "tc.get", "uuid.uuid4", "FunctionCall", "_coerce_tool_arguments" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(output_text, None)", "parse_tool_calls(output_text, request_dict)", "(result.content or '', tool_calls)", "(fallback_text, fallback_calls)", "(result.content, None)", "(fallback_text, None)" ], "awaits": false, "yields": false }, { "name": "_apply_response_format_or_raise", "qualname": "_apply_response_format_or_raise", "full_name": "vllm_mlx.server._apply_response_format_or_raise", "kind": "function", "signature": "def _apply_response_format_or_raise(text: str, response_format: object, *, ensure_ascii: bool=False) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "response_format", "kind": "positional or keyword", "annotation": "object", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "ensure_ascii", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." } ], "return_annotation": "str", "docstring": "Return validated JSON content or fail before returning a success response.", "summary": "Return validated JSON content or fail before returning a success response.", "implementation": "Function `_apply_response_format_or_raise` calls `apply_response_format_or_error`, `HTTPException`, `_strip_backslash_before_unicode`; can raise `HTTPException`; returns `_strip_backslash_before_unicode(text)`.", "documented": true, "public": false, "addressable": true, "line": 1933, "end_line": 1952, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1933-L1952", "decorators": [], "calls": [ "apply_response_format_or_error", "HTTPException", "_strip_backslash_before_unicode" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "_strip_backslash_before_unicode(text)" ], "awaits": false, "yields": false }, { "name": "_response_format_type", "qualname": "_response_format_type", "full_name": "vllm_mlx.server._response_format_type", "kind": "function", "signature": "def _response_format_type(response_format: object | None) -> str | None", "parameters": [ { "name": "response_format", "kind": "positional or keyword", "annotation": "object | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "", "summary": "Function `_response_format_type` calls `isinstance`, `response_format.get`, `getattr`; has 3 explicit return paths.", "implementation": "Function `_response_format_type` calls `isinstance`, `response_format.get`, `getattr`; has 3 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 1955, "end_line": 1960, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1955-L1960", "decorators": [], "calls": [ "isinstance", "response_format.get", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "response_format.get('type')", "getattr(response_format, 'type', None)" ], "awaits": false, "yields": false }, { "name": "_promote_streaming_response_format_delta", "qualname": "_promote_streaming_response_format_delta", "full_name": "vllm_mlx.server._promote_streaming_response_format_delta", "kind": "function", "signature": "def _promote_streaming_response_format_delta(content: str | None, reasoning: str | None, request: ChatCompletionRequest) -> tuple[str | None, str | None]", "parameters": [ { "name": "content", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "reasoning", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str | None, str | None]", "docstring": "Keep response_format JSON on the streaming content channel.\n\nSome thinking parsers classify direct JSON output as reasoning when the\nmodel emits JSON without an explicit reasoning end marker. For\nresponse_format requests, that JSON is the final assistant content.", "summary": "Keep response_format JSON on the streaming content channel.", "implementation": "Function `_promote_streaming_response_format_delta` calls `_response_format_type`, `getattr`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1963, "end_line": 1981, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1963-L1981", "decorators": [], "calls": [ "_response_format_type", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(content, reasoning)", "(reasoning, None)" ], "awaits": false, "yields": false }, { "name": "_new_response_item_id", "qualname": "_new_response_item_id", "full_name": "vllm_mlx.server._new_response_item_id", "kind": "function", "signature": "def _new_response_item_id(prefix: str) -> str", "parameters": [ { "name": "prefix", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Generate stable OpenAI-style item ids.", "summary": "Generate stable OpenAI-style item ids.", "implementation": "Function `_new_response_item_id` calls `uuid.uuid4`; returns `f'{prefix}_{uuid.uuid4().hex}'`.", "documented": true, "public": false, "addressable": true, "line": 1984, "end_line": 1986, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1984-L1986", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'{prefix}_{uuid.uuid4().hex}'" ], "awaits": false, "yields": false }, { "name": "_response_content_to_text", "qualname": "_response_content_to_text", "full_name": "vllm_mlx.server._response_content_to_text", "kind": "function", "signature": "def _response_content_to_text(content) -> str", "parameters": [ { "name": "content", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Normalize Responses API content items into plain text.", "summary": "Normalize Responses API content items into plain text.", "implementation": "Function `_response_content_to_text` calls `isinstance`, `part.get`, `getattr`, `text_parts.append`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1989, "end_line": 2006, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L1989-L2006", "decorators": [], "calls": [ "isinstance", "part.get", "getattr", "text_parts.append", "'\\n'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''", "content", "'\\n'.join((part for part in text_parts if part))" ], "awaits": false, "yields": false }, { "name": "_responses_tools_to_chat_tools", "qualname": "_responses_tools_to_chat_tools", "full_name": "vllm_mlx.server._responses_tools_to_chat_tools", "kind": "function", "signature": "def _responses_tools_to_chat_tools(tools: list[ResponseFunctionTool | dict]) -> tuple[list[dict] | None, list[str]]", "parameters": [ { "name": "tools", "kind": "positional or keyword", "annotation": "list[ResponseFunctionTool | dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[list[dict] | None, list[str]]", "docstring": "Convert supported Responses tools and report unsupported tool types.", "summary": "Convert supported Responses tools and report unsupported tool types.", "implementation": "Function `_responses_tools_to_chat_tools` calls `isinstance`, `tool.get`, `unsupported.append`, `type`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2009, "end_line": 2049, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2009-L2049", "decorators": [], "calls": [ "isinstance", "tool.get", "unsupported.append", "type", "supported.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(None, [])", "(supported or None, unsupported)" ], "awaits": false, "yields": false }, { "name": "_responses_input_to_chat_messages", "qualname": "_responses_input_to_chat_messages", "full_name": "vllm_mlx.server._responses_input_to_chat_messages", "kind": "function", "signature": "def _responses_input_to_chat_messages(request: ResponsesRequest) -> list[dict]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Convert Responses API input items into chat-completions-style messages.", "summary": "Convert Responses API input items into chat-completions-style messages.", "implementation": "Function `_responses_input_to_chat_messages` calls `_responses_store.get`, `HTTPException`, `messages.extend`, `copy.deepcopy`; can raise `HTTPException`; returns `messages`.", "documented": true, "public": false, "addressable": true, "line": 2052, "end_line": 2170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2052-L2170", "decorators": [], "calls": [ "_responses_store.get", "HTTPException", "messages.extend", "copy.deepcopy", "messages.append", "isinstance", "item.get", "_response_content_to_text", "_new_response_item_id", "'\\n'.join", "p.get", "logger.info", "getattr", "type" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "messages" ], "awaits": false, "yields": false }, { "name": "_responses_request_to_new_persisted_messages", "qualname": "_responses_request_to_new_persisted_messages", "full_name": "vllm_mlx.server._responses_request_to_new_persisted_messages", "kind": "function", "signature": "def _responses_request_to_new_persisted_messages(request: ResponsesRequest) -> list[dict]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Persist only the current request's replayable input items.", "summary": "Persist only the current request's replayable input items.", "implementation": "Function `_responses_request_to_new_persisted_messages` calls `request.model_copy`, `_responses_input_to_chat_messages`; returns `_responses_input_to_chat_messages(request_without_history)`.", "documented": true, "public": false, "addressable": true, "line": 2173, "end_line": 2181, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2173-L2181", "decorators": [], "calls": [ "request.model_copy", "_responses_input_to_chat_messages" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_responses_input_to_chat_messages(request_without_history)" ], "awaits": false, "yields": false }, { "name": "_responses_request_to_persisted_messages", "qualname": "_responses_request_to_persisted_messages", "full_name": "vllm_mlx.server._responses_request_to_persisted_messages", "kind": "function", "signature": "def _responses_request_to_persisted_messages(request: ResponsesRequest) -> list[dict]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Persist replayable history for chained previous_response_id requests.\n\nResponses `instructions` are intentionally not replayed across\n`previous_response_id`, but replayable message items are.", "summary": "Persist replayable history for chained previous_response_id requests.", "implementation": "Function `_responses_request_to_persisted_messages` calls `_responses_store.get`, `HTTPException`, `messages.extend`, `copy.deepcopy`; can raise `HTTPException`; returns `messages`.", "documented": true, "public": false, "addressable": true, "line": 2184, "end_line": 2200, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2184-L2200", "decorators": [], "calls": [ "_responses_store.get", "HTTPException", "messages.extend", "copy.deepcopy", "_responses_request_to_new_persisted_messages" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "messages" ], "awaits": false, "yields": false }, { "name": "_responses_request_to_chat_request", "qualname": "_responses_request_to_chat_request", "full_name": "vllm_mlx.server._responses_request_to_chat_request", "kind": "function", "signature": "def _responses_request_to_chat_request(request: ResponsesRequest) -> ChatCompletionRequest", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "ChatCompletionRequest", "docstring": "Build a ChatCompletionRequest from a ResponsesRequest.", "summary": "Build a ChatCompletionRequest from a ResponsesRequest.", "implementation": "Function `_responses_request_to_chat_request` calls `HTTPException`, `logger.debug`, `_responses_tools_to_chat_tools`, `_responses_input_to_chat_messages`; can raise `HTTPException`; returns `ChatCompletionRequest(model=request.model, messages=[Message(**msg) for msg in messages], temperature=request.temperatu…`.", "documented": true, "public": false, "addressable": true, "line": 2203, "end_line": 2253, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2203-L2253", "decorators": [], "calls": [ "HTTPException", "logger.debug", "_responses_tools_to_chat_tools", "_responses_input_to_chat_messages", "', '.join", "sorted", "set", "messages.insert", "msg.get", "'\\n\\n'.join", "str(msg.get('content', '')).strip", "str", "ChatCompletionRequest", "Message" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "ChatCompletionRequest(model=request.model, messages=[Message(**msg) for msg in messages], temperature=request.temperatu…" ], "awaits": false, "yields": false }, { "name": "_build_responses_output_items", "qualname": "_build_responses_output_items", "full_name": "vllm_mlx.server._build_responses_output_items", "kind": "function", "signature": "def _build_responses_output_items(text: str | None, reasoning: str | None, tool_calls: list[ToolCall] | None) -> list[ResponseMessageItem | ResponseReasoningItem | ResponseFunctionCallItem]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "reasoning", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tool_calls", "kind": "positional or keyword", "annotation": "list[ToolCall] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[ResponseMessageItem | ResponseReasoningItem | ResponseFunctionCallItem]", "docstring": "Convert parsed assistant output into Responses API output items.", "summary": "Convert parsed assistant output into Responses API output items.", "implementation": "Function `_build_responses_output_items` calls `output_items.append`, `ResponseReasoningItem`, `_new_response_item_id`, `ResponseReasoningTextPart`; returns `output_items`.", "documented": true, "public": false, "addressable": true, "line": 2256, "end_line": 2293, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2256-L2293", "decorators": [], "calls": [ "output_items.append", "ResponseReasoningItem", "_new_response_item_id", "ResponseReasoningTextPart", "ResponseMessageItem", "ResponseTextContentPart", "ResponseFunctionCallItem" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "output_items" ], "awaits": false, "yields": false }, { "name": "_response_output_items_to_chat_messages", "qualname": "_response_output_items_to_chat_messages", "full_name": "vllm_mlx.server._response_output_items_to_chat_messages", "kind": "function", "signature": "def _response_output_items_to_chat_messages(output_items: list) -> list[dict]", "parameters": [ { "name": "output_items", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Persist assistant output in chat-completions form for previous_response_id.", "summary": "Persist assistant output in chat-completions form for previous_response_id.", "implementation": "Function `_response_output_items_to_chat_messages` calls `isinstance`, `assistant_text_parts.append`, `_response_content_to_text`, `assistant_tool_calls.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2296, "end_line": 2325, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2296-L2325", "decorators": [], "calls": [ "isinstance", "assistant_text_parts.append", "_response_content_to_text", "assistant_tool_calls.append", "''.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "[{'role': 'assistant', 'content': ''.join(assistant_text_parts), 'tool_calls': assistant_tool_calls or None}]" ], "awaits": false, "yields": false }, { "name": "_build_response_object", "qualname": "_build_response_object", "full_name": "vllm_mlx.server._build_response_object", "kind": "function", "signature": "def _build_response_object(request: ResponsesRequest, output_items: list[ResponseMessageItem | ResponseReasoningItem | ResponseFunctionCallItem], prompt_tokens: int, completion_tokens: int, finish_reason: str | None, response_id: str | None=None) -> ResponseObject", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "output_items", "kind": "positional or keyword", "annotation": "list[ResponseMessageItem | ResponseReasoningItem | ResponseFunctionCallItem]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "completion_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "finish_reason", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "response_id", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ResponseObject", "docstring": "Build a full Responses API object.", "summary": "Build a full Responses API object.", "implementation": "Function `_build_response_object` calls `ResponseObject`, `_new_response_item_id`, `_resolve_top_p`, `_resolve_temperature`; returns `response`.", "documented": true, "public": false, "addressable": true, "line": 2328, "end_line": 2367, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2328-L2367", "decorators": [], "calls": [ "ResponseObject", "_new_response_item_id", "_resolve_top_p", "_resolve_temperature", "ResponsesUsage", "ResponseIncompleteDetails" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "response" ], "awaits": false, "yields": false }, { "name": "_prepare_responses_request", "qualname": "_prepare_responses_request", "full_name": "vllm_mlx.server._prepare_responses_request", "kind": "function", "signature": "def _prepare_responses_request(request: ResponsesRequest, *, validate_remote_media: bool=True) -> tuple[BaseEngine, ChatCompletionRequest, list[dict], dict]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "validate_remote_media", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "tuple[BaseEngine, ChatCompletionRequest, list[dict], dict]", "docstring": "Prepare a Responses request for execution on the chat engine.", "summary": "Prepare a Responses request for execution on the chat engine.", "implementation": "Function `_prepare_responses_request` calls `_validate_model_name`, `get_engine`, `_responses_request_to_chat_request`, `logger.info`; returns `(engine, chat_request, messages, chat_kwargs)`.", "documented": true, "public": false, "addressable": true, "line": 2370, "end_line": 2414, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2370-L2414", "decorators": [], "calls": [ "_validate_model_name", "get_engine", "_responses_request_to_chat_request", "logger.info", "isinstance", "len", "_validate_remote_media_urls", "extract_multimodal_content", "canonicalize_system_messages", "_resolve_temperature", "_resolve_top_p", "_resolve_chat_template_kwargs", "convert_tools_for_template" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(engine, chat_request, messages, chat_kwargs)" ], "awaits": false, "yields": false }, { "name": "_prepare_streaming_responses_request", "qualname": "_prepare_streaming_responses_request", "full_name": "vllm_mlx.server._prepare_streaming_responses_request", "kind": "function", "signature": "def _prepare_streaming_responses_request(request: ResponsesRequest) -> tuple[BaseEngine, ChatCompletionRequest, list[dict], dict]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[BaseEngine, ChatCompletionRequest, list[dict], dict]", "docstring": "Prepare a streaming Responses request after eager URL validation.", "summary": "Prepare a streaming Responses request after eager URL validation.", "implementation": "Function `_prepare_streaming_responses_request` calls `_prepare_responses_request`; returns `_prepare_responses_request(request, validate_remote_media=False)`.", "documented": true, "public": false, "addressable": true, "line": 2417, "end_line": 2421, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2417-L2421", "decorators": [], "calls": [ "_prepare_responses_request" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_prepare_responses_request(request, validate_remote_media=False)" ], "awaits": false, "yields": false }, { "name": "_run_responses_request", "qualname": "_run_responses_request", "full_name": "vllm_mlx.server._run_responses_request", "kind": "function", "signature": "async def _run_responses_request(request: ResponsesRequest, raw_request: Request) -> tuple[ResponseObject | None, list[dict]]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[ResponseObject | None, list[dict]]", "docstring": "Execute a Responses API request against the backend chat engine.", "summary": "Execute a Responses API request against the backend chat engine.", "implementation": "Function `_run_responses_request` calls `_prepare_responses_request`, `_wait_with_disconnect`, `engine.chat`, `_parse_tool_calls_with_parser`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2424, "end_line": 2477, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2424-L2477", "decorators": [], "calls": [ "_prepare_responses_request", "_wait_with_disconnect", "engine.chat", "_parse_tool_calls_with_parser", "_reasoning_parser.extract_reasoning", "_build_responses_output_items", "clean_output_text", "_build_response_object", "_responses_request_to_persisted_messages", "persisted_messages.extend", "_response_output_items_to_chat_messages", "copy.deepcopy", "response_object.model_copy", "len", "_responses_store.popitem" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(None, [])", "(response_object, persisted_messages)" ], "awaits": true, "yields": false }, { "name": "_stream_responses_request", "qualname": "_stream_responses_request", "full_name": "vllm_mlx.server._stream_responses_request", "kind": "function", "signature": "async def _stream_responses_request(request: ResponsesRequest) -> AsyncIterator[str]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "AsyncIterator[str]", "docstring": "Execute a Responses API request and stream SSE events incrementally.", "summary": "Execute a Responses API request and stream SSE events incrementally.", "implementation": "Function `_stream_responses_request` calls `_prepare_streaming_responses_request`, `chat_request.model_dump`, `_new_response_item_id`, `_build_response_object`; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 2480, "end_line": 2868, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2480-L2868", "decorators": [], "calls": [ "_prepare_streaming_responses_request", "chat_request.model_dump", "_new_response_item_id", "_build_response_object", "_responses_sse_event", "ResponseCreatedEvent", "ResponseInProgressEvent", "_prepare_streaming_reasoning_parser", "_get_streaming_tool_parser", "engine.stream_chat", "hasattr", "reasoning_parser.extract_reasoning_streaming", "_start_reasoning_item", "ResponseReasoningTextDeltaEvent", "_start_text_item", "ResponseOutputTextDeltaEvent", "SPECIAL_TOKENS_PATTERN.sub", "_streaming_tool_markup_possible_after_delta", "_extract_streaming_tool_delta", "tool_result.get", "_parse_tool_calls_with_parser", "clean_output_text", "ResponseReasoningItem", "ResponseReasoningTextPart", "ResponseReasoningTextDoneEvent", "ResponseContentPartDoneEvent", "ResponseOutputItemDoneEvent", "ResponseMessageItem", "ResponseTextContentPart", "ResponseOutputTextDoneEvent", "ResponseFunctionCallItem", "function_call_items.append", "ResponseOutputItemAddedEvent", "item.model_copy", "ResponseFunctionCallArgumentsDeltaEvent", "output_items.append", "output_items.extend", "_responses_request_to_persisted_messages", "persisted_messages.extend", "_response_output_items_to_chat_messages", "copy.deepcopy", "response_object.model_copy", "len", "_responses_store.popitem", "ResponseCompletedEvent" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "_start_text_item", "qualname": "_stream_responses_request._start_text_item", "full_name": "vllm_mlx.server._stream_responses_request._start_text_item", "kind": "nested function", "signature": "def _start_text_item() -> list[str]", "parameters": [], "return_annotation": "list[str]", "docstring": "", "summary": "Nested Function `_stream_responses_request._start_text_item` calls `_new_response_item_id`, `events.append`, `_responses_sse_event`, `ResponseOutputItemAddedEvent`; returns `events`.", "implementation": "Nested Function `_stream_responses_request._start_text_item` calls `_new_response_item_id`, `events.append`, `_responses_sse_event`, `ResponseOutputItemAddedEvent`; returns `events`.", "documented": false, "public": false, "addressable": false, "line": 2525, "end_line": 2561, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2525-L2561", "decorators": [], "calls": [ "_new_response_item_id", "events.append", "_responses_sse_event", "ResponseOutputItemAddedEvent", "ResponseMessageItem", "ResponseContentPartAddedEvent", "ResponseTextContentPart" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "events" ], "awaits": false, "yields": false }, { "name": "_start_reasoning_item", "qualname": "_stream_responses_request._start_reasoning_item", "full_name": "vllm_mlx.server._stream_responses_request._start_reasoning_item", "kind": "nested function", "signature": "def _start_reasoning_item() -> list[str]", "parameters": [], "return_annotation": "list[str]", "docstring": "", "summary": "Nested Function `_stream_responses_request._start_reasoning_item` calls `_new_response_item_id`, `events.append`, `_responses_sse_event`, `ResponseOutputItemAddedEvent`; returns `events`.", "implementation": "Nested Function `_stream_responses_request._start_reasoning_item` calls `_new_response_item_id`, `events.append`, `_responses_sse_event`, `ResponseOutputItemAddedEvent`; returns `events`.", "documented": false, "public": false, "addressable": false, "line": 2563, "end_line": 2598, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2563-L2598", "decorators": [], "calls": [ "_new_response_item_id", "events.append", "_responses_sse_event", "ResponseOutputItemAddedEvent", "ResponseReasoningItem", "ResponseContentPartAddedEvent", "ResponseReasoningTextPart" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "events" ], "awaits": false, "yields": false }, { "name": "_responses_sse_event", "qualname": "_responses_sse_event", "full_name": "vllm_mlx.server._responses_sse_event", "kind": "function", "signature": "def _responses_sse_event(event_type: str, payload: BaseModel | dict) -> str", "parameters": [ { "name": "event_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "payload", "kind": "positional or keyword", "annotation": "BaseModel | dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Encode a Responses API SSE event.", "summary": "Encode a Responses API SSE event.", "implementation": "Function `_responses_sse_event` calls `isinstance`, `payload.model_dump_json`, `json.dumps`; returns `f'event: {event_type}\\ndata: {data}\\n\\n'`.", "documented": true, "public": false, "addressable": true, "line": 2871, "end_line": 2878, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2871-L2878", "decorators": [], "calls": [ "isinstance", "payload.model_dump_json", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'event: {event_type}\\ndata: {data}\\n\\n'" ], "awaits": false, "yields": false }, { "name": "_strip_harmony_analysis_blocks", "qualname": "_strip_harmony_analysis_blocks", "full_name": "vllm_mlx.server._strip_harmony_analysis_blocks", "kind": "function", "signature": "def _strip_harmony_analysis_blocks(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Remove harmony analysis-channel blocks (and their content) so reasoning\ntext is never handed to the tool parser, while commentary/final text is\npreserved.", "summary": "Remove harmony analysis-channel blocks (and their content) so reasoning text is never handed to the tool parser, while commentary/final text is preserved.", "implementation": "Function `_strip_harmony_analysis_blocks` calls `_HARMONY_ANALYSIS_BLOCK_RE.sub`; returns `_HARMONY_ANALYSIS_BLOCK_RE.sub('', text)`.", "documented": true, "public": false, "addressable": true, "line": 2888, "end_line": 2892, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2888-L2892", "decorators": [], "calls": [ "_HARMONY_ANALYSIS_BLOCK_RE.sub" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_HARMONY_ANALYSIS_BLOCK_RE.sub('', text)" ], "awaits": false, "yields": false }, { "name": "_extract_reasoning_and_tool_calls", "qualname": "_extract_reasoning_and_tool_calls", "full_name": "vllm_mlx.server._extract_reasoning_and_tool_calls", "kind": "function", "signature": "def _extract_reasoning_and_tool_calls(output_text: str, request: ChatCompletionRequest | None=None, *, allow_reasoning: bool=True, engine: BaseEngine | None=None) -> tuple[str | None, str | None, list[ToolCall] | None]", "parameters": [ { "name": "output_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "allow_reasoning", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." }, { "name": "engine", "kind": "keyword-only", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "tuple[str | None, str | None, list[ToolCall] | None]", "docstring": "Extract reasoning first, then parse tool calls from the cleaned content.\n\nNon-streaming responses can contain both a reasoning block and structured\ntool calls in the same final output. If tool parsing runs first and the\nresponse contains tools, the caller can no longer reliably recover the\nreasoning segment because the usual response path skips reasoning parsing\nonce tool_calls is truthy.", "summary": "Extract reasoning first, then parse tool calls from the cleaned content.", "implementation": "Function `_extract_reasoning_and_tool_calls` calls `_reasoning_parser.extract_reasoning`, `getattr`, `_strip_harmony_analysis_blocks`, `_parse_tool_calls_with_parser`; returns `(reasoning_text, cleaned_text, tool_calls)`.", "documented": true, "public": false, "addressable": true, "line": 2895, "end_line": 2951, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2895-L2951", "decorators": [], "calls": [ "_reasoning_parser.extract_reasoning", "getattr", "_strip_harmony_analysis_blocks", "_parse_tool_calls_with_parser", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(reasoning_text, cleaned_text, tool_calls)" ], "awaits": false, "yields": false }, { "name": "_detect_native_tool_support", "qualname": "_detect_native_tool_support", "full_name": "vllm_mlx.server._detect_native_tool_support", "kind": "function", "signature": "def _detect_native_tool_support() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Detect if the active tool parser supports native tool format.\n\nNative format means role=\"tool\" messages and tool_calls fields\nare preserved instead of being converted to text.\n\nReturns:\n True if native format should be preserved", "summary": "Detect if the active tool parser supports native tool format.", "implementation": "Function `_detect_native_tool_support` calls `ToolParserManager.get_tool_parser`, `parser_cls.supports_native_format`, `logger.error`, `ToolParserManager.list_registered`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2954, "end_line": 2983, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2954-L2983", "decorators": [], "calls": [ "ToolParserManager.get_tool_parser", "parser_cls.supports_native_format", "logger.error", "ToolParserManager.list_registered", "logger.warning", "_sanitize_log_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "parser_cls.supports_native_format()" ], "awaits": false, "yields": false }, { "name": "_detect_harmony_rendering", "qualname": "_detect_harmony_rendering", "full_name": "vllm_mlx.server._detect_harmony_rendering", "kind": "function", "signature": "def _detect_harmony_rendering() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Detect whether the harmony rendering path should handle prompt building.\n\nReturns True when ALL of:\n- ``--tool-call-parser`` is set to ``harmony`` or ``gpt-oss``\n- ``--enable-auto-tool-choice`` is on\n- the optional ``openai-harmony`` Python package is importable\n\nThe third condition keeps non-gpt-oss deployments free of an extra\nruntime dependency: if the package isn't installed, the engine falls\nback to the standard ``tokenizer.apply_chat_template`` path. The\nHarmonyToolParser's existing text-flatten behavior also stays in force\nin that fallback so the response side is unchanged.", "summary": "Detect whether the harmony rendering path should handle prompt building.", "implementation": "Function `_detect_harmony_rendering` calls `is_harmony_parser_name`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 2986, "end_line": 3019, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L2986-L3019", "decorators": [], "calls": [ "is_harmony_parser_name", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_tool_choice_disabled", "qualname": "_tool_choice_disabled", "full_name": "vllm_mlx.server._tool_choice_disabled", "kind": "function", "signature": "def _tool_choice_disabled(request: ChatCompletionRequest | None) -> bool", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True when tool_choice explicitly disables tool calling.", "summary": "Return True when tool_choice explicitly disables tool calling.", "implementation": "Function `_tool_choice_disabled` calls `getattr`, `request.model_dump`, `request_dict.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 3022, "end_line": 3031, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3022-L3031", "decorators": [], "calls": [ "getattr", "request.model_dump", "request_dict.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "tool_choice == 'none'" ], "awaits": false, "yields": false }, { "name": "_get_streaming_tool_parser", "qualname": "_get_streaming_tool_parser", "full_name": "vllm_mlx.server._get_streaming_tool_parser", "kind": "function", "signature": "def _get_streaming_tool_parser(request: ChatCompletionRequest | None, engine: BaseEngine | None=None)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Get a streaming-capable tool parser for this request.\n\nUses the configured parser when auto tool choice is enabled, otherwise falls\nback to the generic auto parser so streaming still matches the generic\nnon-streaming tool parsing behavior.", "summary": "Get a streaming-capable tool parser for this request.", "implementation": "Function `_get_streaming_tool_parser` calls `_tool_choice_disabled`, `_get_engine_tokenizer`, `_build_tool_parser`, `logger.warning`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 3034, "end_line": 3071, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3034-L3071", "decorators": [], "calls": [ "_tool_choice_disabled", "_get_engine_tokenizer", "_build_tool_parser", "logger.warning", "_sanitize_log_text", "getattr", "ToolParserManager.get_tool_parser", "parser_cls", "parser.reset" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "_build_tool_parser(engine)", "parser" ], "awaits": false, "yields": false }, { "name": "_extract_streaming_tool_delta", "qualname": "_extract_streaming_tool_delta", "full_name": "vllm_mlx.server._extract_streaming_tool_delta", "kind": "function", "signature": "def _extract_streaming_tool_delta(parser, previous_text: str, delta_text: str, request_context: dict) -> tuple[str, dict | None]", "parameters": [ { "name": "parser", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request_context", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, dict | None]", "docstring": "Parse one request-local streaming delta and return new accumulated text.", "summary": "Parse one request-local streaming delta and return new accumulated text.", "implementation": "Function `_extract_streaming_tool_delta` calls `parser.extract_tool_calls_streaming`; returns `(current_text, result)`.", "documented": true, "public": false, "addressable": true, "line": 3074, "end_line": 3088, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3074-L3088", "decorators": [], "calls": [ "parser.extract_tool_calls_streaming" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(current_text, result)" ], "awaits": false, "yields": false }, { "name": "_stream_request_metadata", "qualname": "_stream_request_metadata", "full_name": "vllm_mlx.server._stream_request_metadata", "kind": "function", "signature": "def _stream_request_metadata(request: ChatCompletionRequest) -> tuple[dict, list | None, bool]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[dict, list | None, bool]", "docstring": "", "summary": "Function `_stream_request_metadata` calls `request.model_dump(include={'tools'}).get`, `request.model_dump`, `bool`; returns `({'tools': tools or []}, tools, include_usage)`.", "implementation": "Function `_stream_request_metadata` calls `request.model_dump(include={'tools'}).get`, `request.model_dump`, `bool`; returns `({'tools': tools or []}, tools, include_usage)`.", "documented": false, "public": false, "addressable": true, "line": 3091, "end_line": 3100, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3091-L3100", "decorators": [], "calls": [ "request.model_dump(include={'tools'}).get", "request.model_dump", "bool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "({'tools': tools or []}, tools, include_usage)" ], "awaits": false, "yields": false }, { "name": "_parse_streaming_tool_content", "qualname": "_parse_streaming_tool_content", "full_name": "vllm_mlx.server._parse_streaming_tool_content", "kind": "function", "signature": "def _parse_streaming_tool_content(parser, accumulated_text: str, delta_text: str, request_context: dict) -> tuple[str, dict | None, bool]", "parameters": [ { "name": "parser", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "accumulated_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request_context", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, dict | None, bool]", "docstring": "", "summary": "Function `_parse_streaming_tool_content` calls `_extract_streaming_tool_delta`; returns `(accumulated_text, result, suppress)`.", "implementation": "Function `_parse_streaming_tool_content` calls `_extract_streaming_tool_delta`; returns `(accumulated_text, result, suppress)`.", "documented": false, "public": false, "addressable": true, "line": 3103, "end_line": 3116, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3103-L3116", "decorators": [], "calls": [ "_extract_streaming_tool_delta" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(accumulated_text, result, suppress)" ], "awaits": false, "yields": false }, { "name": "_streaming_tool_markup_possible", "qualname": "_streaming_tool_markup_possible", "full_name": "vllm_mlx.server._streaming_tool_markup_possible", "kind": "function", "signature": "def _streaming_tool_markup_possible(text: str) -> bool", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Heuristic marker check to avoid parser work on ordinary text chunks.", "summary": "Heuristic marker check to avoid parser work on ordinary text chunks.", "implementation": "Function `_streaming_tool_markup_possible` calls `any`, `_STREAMING_BARE_BRACKET_MARKER.search`, `_STREAMING_BARE_BRACKET_PARTIAL.search`; returns `any((marker in text for marker in _STREAMING_TOOL_MARKERS)) or _STREAMING_BARE_BRACKET_MARKER.search(text) is not None …`.", "documented": true, "public": false, "addressable": true, "line": 3119, "end_line": 3125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3119-L3125", "decorators": [], "calls": [ "any", "_STREAMING_BARE_BRACKET_MARKER.search", "_STREAMING_BARE_BRACKET_PARTIAL.search" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "any((marker in text for marker in _STREAMING_TOOL_MARKERS)) or _STREAMING_BARE_BRACKET_MARKER.search(text) is not None …" ], "awaits": false, "yields": false }, { "name": "_streaming_tool_markup_possible_after_delta", "qualname": "_streaming_tool_markup_possible_after_delta", "full_name": "vllm_mlx.server._streaming_tool_markup_possible_after_delta", "kind": "function", "signature": "def _streaming_tool_markup_possible_after_delta(accumulated_text: str, delta_text: str) -> bool", "parameters": [ { "name": "accumulated_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check only the boundary window needed to detect newly appearing tool markup.\n\nStreaming paths call this before any marker has been seen. Scanning the full\naccumulated text on every ordinary chunk is quadratic for long responses, so\nkeep enough trailing context to catch markers split across chunk boundaries.\nOnce markup is possible, callers switch to the parser path with the full\naccumulated text.", "summary": "Check only the boundary window needed to detect newly appearing tool markup.", "implementation": "Function `_streaming_tool_markup_possible_after_delta` calls `_streaming_tool_markup_possible`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 3128, "end_line": 3143, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3128-L3143", "decorators": [], "calls": [ "_streaming_tool_markup_possible" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "_streaming_tool_markup_possible(check_text)" ], "awaits": false, "yields": false }, { "name": "load_embedding_model", "qualname": "load_embedding_model", "full_name": "vllm_mlx.server.load_embedding_model", "kind": "function", "signature": "def load_embedding_model(model_name: str | None, *, lock: bool=False, reuse_existing: bool=True) -> None", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "lock", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." }, { "name": "reuse_existing", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "None", "docstring": "Load or reuse the embedding model engine when configured.", "summary": "Load or reuse the embedding model engine when configured.", "implementation": "Function `load_embedding_model` calls `EmbeddingEngine`, `_embedding_engine.load`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 3146, "end_line": 3171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3146-L3171", "decorators": [], "calls": [ "EmbeddingEngine", "_embedding_engine.load" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "load_reranker_model", "qualname": "load_reranker_model", "full_name": "vllm_mlx.server.load_reranker_model", "kind": "function", "signature": "def load_reranker_model(model_name: str | None, *, lock: bool=False, reuse_existing: bool=True) -> None", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "lock", "kind": "keyword-only", "annotation": "bool", "default": "False", "required": false, "description": "Optional keyword-only input; defaults to `False`." }, { "name": "reuse_existing", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "None", "docstring": "Load or reuse the reranker model engine when configured.", "summary": "Load or reuse the reranker model engine when configured.", "implementation": "Function `load_reranker_model` calls `RerankEngine`, `_rerank_engine.load`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 3174, "end_line": 3199, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3174-L3199", "decorators": [], "calls": [ "RerankEngine", "_rerank_engine.load" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "load_model", "qualname": "load_model", "full_name": "vllm_mlx.server.load_model", "kind": "function", "signature": "def load_model(model_name: str, use_batching: bool=False, scheduler_config=None, stream_interval: int=1, max_tokens: int=32768, max_request_tokens: int=32768, force_mllm: bool=False, gpu_memory_utilization: float=0.9, served_model_name: str | None=None, trust_remote_code: bool=False, mtp: bool=False, prefill_step_size: int=2048, specprefill_enabled: bool=False, specprefill_threshold: int=8192, specprefill_keep_pct: float=0.3, specprefill_backbone_pct: float=0.0, specprefill_draft_model: str=None, mllm_draft_model: str | None=None, mllm_draft_kind: str | None=None, mllm_draft_block_size: int | None=None, warm_prompts_path: str | None=None, auto_unload_idle_seconds: float=0.0, lazy_load_model: bool=False)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "use_batching", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Use continuous batching (BatchedEngine) vs simple mode (SimpleEngine)" }, { "name": "scheduler_config", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Scheduler config for batched mode" }, { "name": "stream_interval", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Tokens to batch before streaming (batched mode only)" }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "32768", "required": false, "description": "Default max tokens for generation" }, { "name": "max_request_tokens", "kind": "positional or keyword", "annotation": "int", "default": "32768", "required": false, "description": "Maximum max_tokens accepted from API clients" }, { "name": "force_mllm", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Force loading as MLLM even if not auto-detected" }, { "name": "gpu_memory_utilization", "kind": "positional or keyword", "annotation": "float", "default": "0.9", "required": false, "description": "Optional positional or keyword input; defaults to `0.9`." }, { "name": "served_model_name", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "trust_remote_code", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Allow HuggingFace remote code execution during model/tokenizer loading" }, { "name": "mtp", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable native MTP speculative decoding (SimpleEngine only)" }, { "name": "prefill_step_size", "kind": "positional or keyword", "annotation": "int", "default": "2048", "required": false, "description": "Chunk size for prompt prefill processing (default: 2048)" }, { "name": "specprefill_enabled", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Enable SpecPrefill (SimpleEngine only)" }, { "name": "specprefill_threshold", "kind": "positional or keyword", "annotation": "int", "default": "8192", "required": false, "description": "Minimum suffix tokens to trigger SpecPrefill (default: 8192)" }, { "name": "specprefill_keep_pct", "kind": "positional or keyword", "annotation": "float", "default": "0.3", "required": false, "description": "Fraction of tokens to keep (default: 0.3)" }, { "name": "specprefill_backbone_pct", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Fraction of chunks reserved for evenly spaced coverage" }, { "name": "specprefill_draft_model", "kind": "positional or keyword", "annotation": "str", "default": "None", "required": false, "description": "Path to small draft model for SpecPrefill scoring" }, { "name": "mllm_draft_model", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional MLLM speculative draft/assistant model path." }, { "name": "mllm_draft_kind", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional mlx-vlm draft kind, for example \"mtp\"." }, { "name": "mllm_draft_block_size", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional speculative block size passed to mlx-vlm." }, { "name": "warm_prompts_path", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "auto_unload_idle_seconds", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Idle time before auto-unloading the main model. When non-zero, the main model is managed through lifecycle residency instead of being loaded immediately in this function." }, { "name": "lazy_load_model", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "When lifecycle residency is enabled, defer the first resident load until the first request instead of FastAPI lifespan startup." } ], "return_annotation": "not annotated", "docstring": "Load a model (auto-detects MLLM vs LLM).\n\nArgs:\n model_name: HuggingFace model name or local path\n use_batching: Use continuous batching (BatchedEngine) vs simple mode (SimpleEngine)\n scheduler_config: Scheduler config for batched mode\n stream_interval: Tokens to batch before streaming (batched mode only)\n max_tokens: Default max tokens for generation\n max_request_tokens: Maximum max_tokens accepted from API clients\n force_mllm: Force loading as MLLM even if not auto-detected\n trust_remote_code: Allow HuggingFace remote code execution during model/tokenizer loading\n mtp: Enable native MTP speculative decoding (SimpleEngine only)\n prefill_step_size: Chunk size for prompt prefill processing (default: 2048)\n specprefill_enabled: Enable SpecPrefill (SimpleEngine only)\n specprefill_threshold: Minimum suffix tokens to trigger SpecPrefill (default: 8192)\n specprefill_keep_pct: Fraction of tokens to keep (default: 0.3)\n specprefill_backbone_pct: Fraction of chunks reserved for evenly spaced coverage\n specprefill_draft_model: Path to small draft model for SpecPrefill scoring\n mllm_draft_model: Optional MLLM speculative draft/assistant model path.\n mllm_draft_kind: Optional mlx-vlm draft kind, for example \"mtp\".\n mllm_draft_block_size: Optional speculative block size passed to mlx-vlm.\n auto_unload_idle_seconds: Idle time before auto-unloading the main model.\n When non-zero, the main model is managed through lifecycle\n residency instead of being loaded immediately in this function.\n lazy_load_model: When lifecycle residency is enabled, defer the first\n resident load until the first request instead of FastAPI lifespan\n startup.", "summary": "Load a model (auto-detects MLLM vs LLM).", "implementation": "Function `load_model` calls `ValueError`, `RuntimeError`, `getattr`, `isinstance`; can raise `ValueError`, `RuntimeError`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 3202, "end_line": 3431, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3202-L3431", "decorators": [], "calls": [ "ValueError", "RuntimeError", "getattr", "isinstance", "_residency_manager.get_engine", "_residency_manager.get_status", "existing_status.get", "_invalidate_tool_parser_cache", "logger.info", "ModelSpec", "ResidencyManager", "_residency_manager.register_model", "BatchedEngine", "simple_engine_cls", "asyncio.get_event_loop", "asyncio.new_event_loop", "asyncio.set_event_loop", "loop.run_until_complete", "_engine.start", "suppress", "loop.shutdown_default_executor", "loop.close", "previous_loop.is_closed", "_detect_native_tool_support", "_detect_harmony_rendering" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError", "RuntimeError" ], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "load_model_registry", "qualname": "load_model_registry", "full_name": "vllm_mlx.server.load_model_registry", "kind": "function", "signature": "def load_model_registry(config_path: str, *, defaults: RegistryServeDefaults) -> None", "parameters": [ { "name": "config_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "defaults", "kind": "keyword-only", "annotation": "RegistryServeDefaults", "default": "", "required": true, "description": "Required keyword-only input." } ], "return_annotation": "None", "docstring": "Load a registry-backed model manager from YAML configuration.", "summary": "Load a registry-backed model manager from YAML configuration.", "implementation": "Function `load_model_registry` calls `load_registry_config`, `ModelManager`, `logger.info`, `len`.", "documented": true, "public": true, "addressable": true, "line": 3434, "end_line": 3457, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3434-L3457", "decorators": [], "calls": [ "load_registry_config", "ModelManager", "logger.info", "len", "log_memory_budget_report", "build_memory_budget_report" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_usage", "qualname": "get_usage", "full_name": "vllm_mlx.server.get_usage", "kind": "function", "signature": "def get_usage(output: GenerationOutput) -> Usage", "parameters": [ { "name": "output", "kind": "positional or keyword", "annotation": "GenerationOutput", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Usage", "docstring": "Extract usage metrics from GenerationOutput.", "summary": "Extract usage metrics from GenerationOutput.", "implementation": "Function `get_usage` calls `hasattr`, `Usage`; returns `Usage(prompt_tokens=total_prompt_tokens, completion_tokens=total_completion_tokens, total_tokens=total_prompt_tokens + …`.", "documented": true, "public": true, "addressable": true, "line": 3460, "end_line": 3472, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3460-L3472", "decorators": [], "calls": [ "hasattr", "Usage" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Usage(prompt_tokens=total_prompt_tokens, completion_tokens=total_completion_tokens, total_tokens=total_prompt_tokens + …" ], "awaits": false, "yields": false }, { "name": "metrics", "qualname": "metrics", "full_name": "vllm_mlx.server.metrics", "kind": "function", "signature": "async def metrics()", "parameters": [], "return_annotation": "not annotated", "docstring": "Prometheus scrape endpoint (disabled by default).", "summary": "Prometheus scrape endpoint (disabled by default).", "implementation": "Function `metrics` calls `HTTPException`, `_metrics.render_metrics`, `Response`; can raise `HTTPException`; returns `Response(content=payload, headers={'Content-Type': content_type})`.", "documented": true, "public": true, "addressable": true, "line": 3476, "end_line": 3485, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3476-L3485", "decorators": [ "app.get('/metrics')" ], "calls": [ "HTTPException", "_metrics.render_metrics", "Response" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "Response(content=payload, headers={'Content-Type': content_type})" ], "awaits": false, "yields": false }, { "name": "health", "qualname": "health", "full_name": "vllm_mlx.server.health", "kind": "function", "signature": "async def health()", "parameters": [], "return_annotation": "not annotated", "docstring": "Health check endpoint.", "summary": "Health check endpoint.", "implementation": "Function `health` calls `sum`, `_mcp_manager.get_server_status`, `len`, `_mcp_manager.get_all_tools`; returns `payload`.", "documented": true, "public": true, "addressable": true, "line": 3489, "end_line": 3544, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3489-L3544", "decorators": [ "app.get('/health')" ], "calls": [ "sum", "_mcp_manager.get_server_status", "len", "_mcp_manager.get_all_tools", "_engine.get_stats", "_get_lifecycle_status", "lifecycle.get", "_list_available_model_names", "is_mllm_model", "engine_stats.get", "payload.update" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "payload" ], "awaits": false, "yields": false }, { "name": "status", "qualname": "status", "full_name": "vllm_mlx.server.status", "kind": "function", "signature": "async def status()", "parameters": [], "return_annotation": "not annotated", "docstring": "Real-time status with per-request details for debugging and monitoring.", "summary": "Real-time status with per-request details for debugging and monitoring.", "implementation": "Function `status` calls `round`, `_model_manager.list_models`, `_public_lifecycle_status`, `_get_lifecycle_status`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3548, "end_line": 3597, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3548-L3597", "decorators": [ "app.get('/v1/status', dependencies=[Depends(verify_api_key)])" ], "calls": [ "round", "_model_manager.list_models", "_public_lifecycle_status", "_get_lifecycle_status", "_engine.get_stats", "stats.get", "bg.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'status': 'running', 'model_manager': {'memory_budget_gb': round(_model_manager.memory_budget_bytes / 1024 ** 3, 2), '…", "{'status': 'not_loaded', 'model': _model_name, 'residency': lifecycle, 'requests': []}", "{'status': 'running' if stats.get('running') else 'stopped', 'model': _model_name, 'residency': lifecycle, 'uptime_s': …" ], "awaits": false, "yields": false }, { "name": "cache_stats", "qualname": "cache_stats", "full_name": "vllm_mlx.server.cache_stats", "kind": "function", "signature": "async def cache_stats()", "parameters": [], "return_annotation": "not annotated", "docstring": "Get cache statistics for debugging and monitoring.", "summary": "Get cache statistics for debugging and monitoring.", "implementation": "Function `cache_stats` calls `hasattr`, `_engine.get_cache_stats`, `get_multimodal_kv_cache_stats`, `get_pixel_values_cache_stats`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3601, "end_line": 3627, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3601-L3627", "decorators": [ "app.get('/v1/cache/stats', dependencies=[Depends(verify_api_key)])" ], "calls": [ "hasattr", "_engine.get_cache_stats", "get_multimodal_kv_cache_stats", "get_pixel_values_cache_stats", "get_pil_cache_stats" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'engine_cache': engine_cache, 'multimodal_kv_cache': get_multimodal_kv_cache_stats(), 'pixel_values_cache': get_pixel_…", "{'engine_cache': engine_cache, 'error': 'Cache stats not available (mlx_vlm not loaded)'}" ], "awaits": false, "yields": false }, { "name": "clear_cache", "qualname": "clear_cache", "full_name": "vllm_mlx.server.clear_cache", "kind": "function", "signature": "async def clear_cache()", "parameters": [], "return_annotation": "not annotated", "docstring": "Clear all caches.", "summary": "Clear all caches.", "implementation": "Function `clear_cache` calls `hasattr`, `_engine.clear_runtime_caches`, `logger.warning`, `str`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3631, "end_line": 3659, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3631-L3659", "decorators": [ "app.delete('/v1/cache', dependencies=[Depends(verify_api_key)])" ], "calls": [ "hasattr", "_engine.clear_runtime_caches", "logger.warning", "str", "clear_multimodal_kv_cache", "clear_pixel_values_cache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'status': 'cleared', 'engine_cache': cleared_engine, 'caches': ['multimodal_kv', 'pixel_values', 'pil_image']}", "{'status': 'cleared', 'engine_cache': cleared_engine, 'error': 'Cache clear not available (mlx_vlm not loaded)'}" ], "awaits": false, "yields": false }, { "name": "clear_prefix_cache", "qualname": "clear_prefix_cache", "full_name": "vllm_mlx.server.clear_prefix_cache", "kind": "function", "signature": "async def clear_prefix_cache()", "parameters": [], "return_annotation": "not annotated", "docstring": "Clear the text prefix cache used for KV reuse in continuous batching.\n\nIf the server was started with ``--warm-prompts``, the warm-up is\nre-run in the background after clear so the next real request still\nhits the cache. Response returns immediately without waiting for\nthe re-warm to finish.", "summary": "Clear the text prefix cache used for KV reuse in continuous batching.", "implementation": "Function `clear_prefix_cache` calls `hasattr`, `_engine.clear_prefix_cache`, `logger.warning`, `_sanitize_log_text`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 3663, "end_line": 3713, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3663-L3713", "decorators": [ "app.delete('/v1/cache/prefix', dependencies=[Depends(verify_api_key)])" ], "calls": [ "hasattr", "_engine.clear_prefix_cache", "logger.warning", "_sanitize_log_text", "asyncio.create_task", "_rewarm" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'status': 'no_engine'}", "{'status': status, 'rewarm_scheduled': rewarm_scheduled}" ], "awaits": false, "yields": false }, { "name": "_rewarm", "qualname": "clear_prefix_cache._rewarm", "full_name": "vllm_mlx.server.clear_prefix_cache._rewarm", "kind": "nested function", "signature": "async def _rewarm()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `clear_prefix_cache._rewarm` calls `load_warmup_file`, `warm_prefix_cache`, `logger.info`, `logger.warning`; awaits asynchronous work.", "implementation": "Nested Function `clear_prefix_cache._rewarm` calls `load_warmup_file`, `warm_prefix_cache`, `logger.info`, `logger.warning`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 3688, "end_line": 3707, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3688-L3707", "decorators": [], "calls": [ "load_warmup_file", "warm_prefix_cache", "logger.info", "logger.warning", "_sanitize_log_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "cancel_request", "qualname": "cancel_request", "full_name": "vllm_mlx.server.cancel_request", "kind": "function", "signature": "async def cancel_request(request_id: str)", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Cancel an active or queued request.\n\nThe request_id is the chatcmpl-xxx ID from the first SSE streaming chunk.", "summary": "Cancel an active or queued request.", "implementation": "Function `cancel_request` calls `get_engine`, `engine.abort_request`, `logger.exception`, `HTTPException`; awaits asynchronous work; can raise `HTTPException`; returns `{'object': 'request.cancel', 'id': request_id, 'cancelled': True, 'model': _model_name}`.", "documented": true, "public": true, "addressable": true, "line": 3720, "end_line": 3747, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3720-L3747", "decorators": [ "app.post('/v1/requests/{request_id}/cancel', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "get_engine", "engine.abort_request", "logger.exception", "HTTPException", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "{'object': 'request.cancel', 'id': request_id, 'cancelled': True, 'model': _model_name}" ], "awaits": true, "yields": false }, { "name": "delete_request", "qualname": "delete_request", "full_name": "vllm_mlx.server.delete_request", "kind": "function", "signature": "async def delete_request(request_id: str)", "parameters": [ { "name": "request_id", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "OpenAI-style alias for cancelling an active or queued request.", "summary": "OpenAI-style alias for cancelling an active or queued request.", "implementation": "Function `delete_request` calls `cancel_request`; awaits asynchronous work; returns `await cancel_request(request_id)`.", "documented": true, "public": true, "addressable": true, "line": 3754, "end_line": 3756, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3754-L3756", "decorators": [ "app.delete('/v1/requests/{request_id}', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "cancel_request" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "await cancel_request(request_id)" ], "awaits": true, "yields": false }, { "name": "list_models", "qualname": "list_models", "full_name": "vllm_mlx.server.list_models", "kind": "function", "signature": "async def list_models() -> ModelsResponse", "parameters": [], "return_annotation": "ModelsResponse", "docstring": "List available models.", "summary": "List available models.", "implementation": "Function `list_models` calls `models.extend`, `ModelInfo`, `_model_manager.list_models`, `models.append`; returns `ModelsResponse(data=models)`.", "documented": true, "public": true, "addressable": true, "line": 3760, "end_line": 3775, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3760-L3775", "decorators": [ "app.get('/v1/models', dependencies=[Depends(verify_api_key)])" ], "calls": [ "models.extend", "ModelInfo", "_model_manager.list_models", "models.append", "ModelsResponse" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "ModelsResponse(data=models)" ], "awaits": false, "yields": false }, { "name": "create_embeddings", "qualname": "create_embeddings", "full_name": "vllm_mlx.server.create_embeddings", "kind": "function", "signature": "async def create_embeddings(request: EmbeddingRequest) -> EmbeddingResponse", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "EmbeddingRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "EmbeddingResponse", "docstring": "Create embeddings for the given input text(s).\n\nOpenAI-compatible embeddings API supporting single or batch inputs.\n\nSingle text:\n```json\n{\n \"model\": \"mlx-community/all-MiniLM-L6-v2-4bit\",\n \"input\": \"The quick brown fox jumps over the lazy dog\"\n}\n```\n\nBatch of texts:\n```json\n{\n \"model\": \"mlx-community/embeddinggemma-300m-6bit\",\n \"input\": [\n \"I love machine learning\",\n \"Deep learning is fascinating\",\n \"Neural networks are powerful\"\n ]\n}\n```\n\nResponse:\n```json\n{\n \"object\": \"list\",\n \"data\": [\n {\"object\": \"embedding\", \"index\": 0, \"embedding\": [0.023, -0.982, ...]},\n {\"object\": \"embedding\", \"index\": 1, \"embedding\": [0.112, -0.543, ...]},\n {\"object\": \"embedding\", \"index\": 2, \"embedding\": [0.876, 0.221, ...]}\n ],\n \"model\": \"mlx-community/embeddinggemma-300m-6bit\",\n \"usage\": {\"prompt_tokens\": 24, \"total_tokens\": 24}\n}\n```\n\nSupported request-time models:\n- mlx-community/all-MiniLM-L6-v2-4bit (fast, compact)\n- mlx-community/embeddinggemma-300m-6bit (high quality)\n- mlx-community/bge-large-en-v1.5-4bit (best for English)\n- mlx-community/multilingual-e5-small-mlx\n- mlx-community/multilingual-e5-large-mlx\n- mlx-community/bert-base-uncased-mlx\n- mlx-community/ModernBERT-base-mlx\n\nOther embedding models must be pinned explicitly with --embedding-model at\nserver startup.", "summary": "Create embeddings for the given input text(s).", "implementation": "Function `create_embeddings` calls `_metrics.track_inference`, `resolve_embedding_model_name`, `load_embedding_model`, `isinstance`; can raise `HTTPException`; returns `response`.", "documented": true, "public": true, "addressable": true, "line": 3787, "end_line": 3908, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3787-L3908", "decorators": [ "app.post('/v1/embeddings', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "_metrics.track_inference", "resolve_embedding_model_name", "load_embedding_model", "isinstance", "HTTPException", "time.perf_counter", "_embedding_engine.count_tokens", "_embedding_engine.embed", "logger.info", "len", "EmbeddingData", "enumerate", "EmbeddingResponse", "EmbeddingUsage", "tracker.finish", "_metrics_result_from_status", "_log_and_raise_internal_error" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "response" ], "awaits": false, "yields": false }, { "name": "rerank_documents", "qualname": "rerank_documents", "full_name": "vllm_mlx.server.rerank_documents", "kind": "function", "signature": "async def rerank_documents(request: RerankRequest) -> RerankResponse", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "RerankRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "RerankResponse", "docstring": "Rerank documents against a query using a cross-encoder model.\n\nJina/Cohere-compatible reranking API. Accepts a query and a list of\ndocuments (strings or {text: ...} objects), returns results sorted\nby relevance score descending.", "summary": "Rerank documents against a query using a cross-encoder model.", "implementation": "Function `rerank_documents` calls `HTTPException`, `request.query.strip`, `len`, `isinstance`; awaits asynchronous work; can raise `HTTPException`; returns `RerankResponse(model=model_name, results=results, usage=RerankUsage(total_tokens=total_tokens))`.", "documented": true, "public": true, "addressable": true, "line": 3920, "end_line": 4038, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L3920-L4038", "decorators": [ "app.post('/v1/rerank', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "HTTPException", "request.query.strip", "len", "isinstance", "doc_texts.append", "original_docs.append", "type", "time.perf_counter", "asyncio.to_thread", "logger.info", "enumerate", "RerankResult", "results.append", "results.sort", "RerankResponse", "RerankUsage", "logger.error", "str" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "RerankResponse(model=model_name, results=results, usage=RerankUsage(total_tokens=total_tokens))" ], "awaits": true, "yields": false }, { "name": "list_mcp_tools", "qualname": "list_mcp_tools", "full_name": "vllm_mlx.server.list_mcp_tools", "kind": "function", "signature": "async def list_mcp_tools() -> MCPToolsResponse", "parameters": [], "return_annotation": "MCPToolsResponse", "docstring": "List all available MCP tools.", "summary": "List all available MCP tools.", "implementation": "Function `list_mcp_tools` calls `MCPToolsResponse`, `_mcp_manager.get_all_tools`, `tools.append`, `MCPToolInfo`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 4047, "end_line": 4063, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4047-L4063", "decorators": [ "app.get('/v1/mcp/tools', dependencies=[Depends(verify_api_key)])" ], "calls": [ "MCPToolsResponse", "_mcp_manager.get_all_tools", "tools.append", "MCPToolInfo", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "MCPToolsResponse(tools=[], count=0)", "MCPToolsResponse(tools=tools, count=len(tools))" ], "awaits": false, "yields": false }, { "name": "list_mcp_servers", "qualname": "list_mcp_servers", "full_name": "vllm_mlx.server.list_mcp_servers", "kind": "function", "signature": "async def list_mcp_servers() -> MCPServersResponse", "parameters": [], "return_annotation": "MCPServersResponse", "docstring": "Get status of all MCP servers.", "summary": "Get status of all MCP servers.", "implementation": "Function `list_mcp_servers` calls `MCPServersResponse`, `_mcp_manager.get_server_status`, `servers.append`, `MCPServerInfo`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 4067, "end_line": 4084, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4067-L4084", "decorators": [ "app.get('/v1/mcp/servers', dependencies=[Depends(verify_api_key)])" ], "calls": [ "MCPServersResponse", "_mcp_manager.get_server_status", "servers.append", "MCPServerInfo" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "MCPServersResponse(servers=[])", "MCPServersResponse(servers=servers)" ], "awaits": false, "yields": false }, { "name": "execute_mcp_tool", "qualname": "execute_mcp_tool", "full_name": "vllm_mlx.server.execute_mcp_tool", "kind": "function", "signature": "async def execute_mcp_tool(request: MCPExecuteRequest) -> MCPExecuteResponse", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "MCPExecuteRequest", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "MCPExecuteResponse", "docstring": "Execute an MCP tool.", "summary": "Execute an MCP tool.", "implementation": "Function `execute_mcp_tool` calls `HTTPException`, `ToolExecutor`, `uuid.uuid4`, `_mcp_executor.execute_tool_calls`; awaits asynchronous work; can raise `HTTPException`; returns `MCPExecuteResponse(tool_name=result.tool_name, content=result.content, is_error=result.is_error, error_message=result.e…`.", "documented": true, "public": true, "addressable": true, "line": 4088, "end_line": 4117, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4088-L4117", "decorators": [ "app.post('/v1/mcp/execute', dependencies=[Depends(verify_api_key)])" ], "calls": [ "HTTPException", "ToolExecutor", "uuid.uuid4", "_mcp_executor.execute_tool_calls", "MCPExecuteResponse" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "MCPExecuteResponse(tool_name=result.tool_name, content=result.content, is_error=result.is_error, error_message=result.e…" ], "awaits": true, "yields": false }, { "name": "create_transcription", "qualname": "create_transcription", "full_name": "vllm_mlx.server.create_transcription", "kind": "function", "signature": "async def create_transcription(file: UploadFile, model: str='whisper-large-v3', language: str | None=None, response_format: str='json')", "parameters": [ { "name": "file", "kind": "positional or keyword", "annotation": "UploadFile", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "'whisper-large-v3'", "required": false, "description": "Optional positional or keyword input; defaults to `'whisper-large-v3'`." }, { "name": "language", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "response_format", "kind": "positional or keyword", "annotation": "str", "default": "'json'", "required": false, "description": "Optional positional or keyword input; defaults to `'json'`." } ], "return_annotation": "not annotated", "docstring": "Transcribe audio to text (OpenAI Whisper API compatible).\n\nSupported models:\n- whisper-large-v3 (multilingual, best quality)\n- whisper-large-v3-turbo (faster)\n- whisper-medium, whisper-small (lighter)\n- parakeet-tdt-0.6b-v2 (English, fastest)", "summary": "Transcribe audio to text (OpenAI Whisper API compatible).", "implementation": "Function `create_transcription` calls `_metrics.track_inference`, `resolve_stt_model_name`, `STTEngine`, `_stt_engine.load`; awaits asynchronous work; can raise `HTTPException`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 4130, "end_line": 4196, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4130-L4196", "decorators": [ "app.post('/v1/audio/transcriptions', dependencies=[Depends(verify_api_key)])" ], "calls": [ "_metrics.track_inference", "resolve_stt_model_name", "STTEngine", "_stt_engine.load", "save_upload_with_limit", "_stt_engine.transcribe", "os.unlink", "tracker.finish", "HTTPException", "_metrics_result_from_status", "_log_and_raise_internal_error" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "result.text", "{'text': result.text, 'language': result.language, 'duration': result.duration}" ], "awaits": true, "yields": false }, { "name": "create_speech", "qualname": "create_speech", "full_name": "vllm_mlx.server.create_speech", "kind": "function", "signature": "async def create_speech(model: str='kokoro', input: str='', voice: str='af_heart', speed: float=1.0, response_format: str='wav')", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "'kokoro'", "required": false, "description": "Optional positional or keyword input; defaults to `'kokoro'`." }, { "name": "input", "kind": "positional or keyword", "annotation": "str", "default": "''", "required": false, "description": "Optional positional or keyword input; defaults to `''`." }, { "name": "voice", "kind": "positional or keyword", "annotation": "str", "default": "'af_heart'", "required": false, "description": "Optional positional or keyword input; defaults to `'af_heart'`." }, { "name": "speed", "kind": "positional or keyword", "annotation": "float", "default": "1.0", "required": false, "description": "Optional positional or keyword input; defaults to `1.0`." }, { "name": "response_format", "kind": "positional or keyword", "annotation": "str", "default": "'wav'", "required": false, "description": "Optional positional or keyword input; defaults to `'wav'`." } ], "return_annotation": "not annotated", "docstring": "Generate speech from text (OpenAI TTS API compatible).\n\nSupported models:\n- kokoro (fast, lightweight)\n- chatterbox (multilingual, expressive)\n- vibevoice (realtime)\n- voxcpm (Chinese/English)", "summary": "Generate speech from text (OpenAI TTS API compatible).", "implementation": "Function `create_speech` calls `_metrics.track_inference`, `resolve_tts_model_name`, `validate_tts_input_length`, `TTSEngine`; can raise `HTTPException`; returns `Response(content=audio_bytes, media_type=content_type)`.", "documented": true, "public": true, "addressable": true, "line": 4200, "end_line": 4254, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4200-L4254", "decorators": [ "app.post('/v1/audio/speech', dependencies=[Depends(verify_api_key)])" ], "calls": [ "_metrics.track_inference", "resolve_tts_model_name", "validate_tts_input_length", "TTSEngine", "_tts_engine.load", "_tts_engine.generate", "_tts_engine.to_bytes", "tracker.finish", "Response", "HTTPException", "_metrics_result_from_status", "_log_and_raise_internal_error" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "Response(content=audio_bytes, media_type=content_type)" ], "awaits": false, "yields": false }, { "name": "list_voices", "qualname": "list_voices", "full_name": "vllm_mlx.server.list_voices", "kind": "function", "signature": "async def list_voices(model: str='kokoro')", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "str", "default": "'kokoro'", "required": false, "description": "Optional positional or keyword input; defaults to `'kokoro'`." } ], "return_annotation": "not annotated", "docstring": "List available voices for a TTS model.", "summary": "List available voices for a TTS model.", "implementation": "Function `list_voices` calls `model.lower`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 4258, "end_line": 4267, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4258-L4267", "decorators": [ "app.get('/v1/audio/voices', dependencies=[Depends(verify_api_key)])" ], "calls": [ "model.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'voices': KOKORO_VOICES}", "{'voices': CHATTERBOX_VOICES}", "{'voices': ['default']}" ], "awaits": false, "yields": false }, { "name": "_ensure_sse_terminal", "qualname": "_ensure_sse_terminal", "full_name": "vllm_mlx.server._ensure_sse_terminal", "kind": "function", "signature": "async def _ensure_sse_terminal(generator: AsyncIterator[str], terminal_frame: str) -> AsyncIterator[str]", "parameters": [ { "name": "generator", "kind": "positional or keyword", "annotation": "AsyncIterator[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "terminal_frame", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "AsyncIterator[str]", "docstring": "Guarantee that *terminal_frame* is emitted exactly once at the end of\n*generator*, even if the generator raises mid-stream.\n\nIf the inner generator already yields the terminal frame on its happy path,\nthe wrapper detects it and avoids double-emission. If the generator raises\nbefore reaching the terminal, the wrapper emits it in the ``finally`` block.", "summary": "Guarantee that *terminal_frame* is emitted exactly once at the end of *generator*, even if the generator raises mid-stream.", "implementation": "Function `_ensure_sse_terminal` calls `logger.error`; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 4275, "end_line": 4296, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4275-L4296", "decorators": [], "calls": [ "logger.error" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "_find_uvicorn_cycle", "qualname": "_find_uvicorn_cycle", "full_name": "vllm_mlx.server._find_uvicorn_cycle", "kind": "function", "signature": "def _find_uvicorn_cycle(obj, depth=0, visited=None)", "parameters": [ { "name": "obj", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "depth", "kind": "positional or keyword", "annotation": "not annotated", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." }, { "name": "visited", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Walk through middleware wrappers to find uvicorn's RequestResponseCycle.\n\nThis relies on uvicorn's internal ``RequestResponseCycle.disconnected``\nattribute and Starlette's middleware closure layout. Tested against\nuvicorn 0.34-0.40 and starlette 0.44-0.46. If either changes the\ninternal layout, this function returns None and disconnect detection\nsilently falls back to timeout-only behaviour.", "summary": "Walk through middleware wrappers to find uvicorn's RequestResponseCycle.", "implementation": "Function `_find_uvicorn_cycle` calls `set`, `id`, `visited.add`, `hasattr`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 4299, "end_line": 4346, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4299-L4346", "decorators": [], "calls": [ "set", "id", "visited.add", "hasattr", "isinstance", "getattr", "_find_uvicorn_cycle" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "obj", "result" ], "awaits": false, "yields": false }, { "name": "_is_client_disconnected", "qualname": "_is_client_disconnected", "full_name": "vllm_mlx.server._is_client_disconnected", "kind": "function", "signature": "def _is_client_disconnected(raw_request: Request) -> bool", "parameters": [ { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Reliable client disconnect check.\n\nStarlette's ``is_disconnected()`` uses an immediately-cancelled\n``anyio.CancelScope`` which prevents the ASGI ``receive()`` from\nexecuting — so it always returns False for non-streaming requests.\n\nThis function bypasses Starlette and reads uvicorn's internal\n``disconnected`` flag directly from the ``RequestResponseCycle``,\nwalking through any middleware wrappers via closures.", "summary": "Reliable client disconnect check.", "implementation": "Function `_is_client_disconnected` calls `getattr`, `_find_uvicorn_cycle`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 4349, "end_line": 4374, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4349-L4374", "decorators": [], "calls": [ "getattr", "_find_uvicorn_cycle" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "_disconnect_guard", "qualname": "_disconnect_guard", "full_name": "vllm_mlx.server._disconnect_guard", "kind": "function", "signature": "async def _disconnect_guard(generator: AsyncIterator[str], raw_request: Request, poll_interval: float=0.5, heartbeat_interval: float=5.0, cleanup=None, timeout: float | None=None) -> AsyncIterator[str]", "parameters": [ { "name": "generator", "kind": "positional or keyword", "annotation": "AsyncIterator[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "poll_interval", "kind": "positional or keyword", "annotation": "float", "default": "0.5", "required": false, "description": "Optional positional or keyword input; defaults to `0.5`." }, { "name": "heartbeat_interval", "kind": "positional or keyword", "annotation": "float", "default": "5.0", "required": false, "description": "Optional positional or keyword input; defaults to `5.0`." }, { "name": "cleanup", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "float | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "AsyncIterator[str]", "docstring": "Wrap streaming generator to abort on client disconnect.\n\nUses asyncio racing: each __anext__() on the inner generator is\nraced against a disconnect poller. When neither completes within\n``heartbeat_interval`` seconds, an SSE comment is yielded as a\nheartbeat. This forces an ASGI write which triggers broken-pipe\ndetection — without heartbeats, ``is_disconnected()`` stays False\nduring long prefill because no data is written to the socket.\n\nIf *timeout* is set, it bounds inactivity from the inner generator,\nnot the total stream lifetime. A stream that continues to produce\nchunks must be allowed to complete even when generation takes longer\nthan the configured interval. Heartbeats force ASGI writes to detect a\ndisconnected client, but do not count as generator progress.\n\nOn disconnect, the cancellation propagates to stream_outputs()\nfinally-block → abort_request() → abort_prefill().", "summary": "Wrap streaming generator to abort on client disconnect.", "implementation": "Function `_disconnect_guard` calls `_time.monotonic`, `logger.info`, `generator.__aiter__`, `asyncio.create_task`; awaits asynchronous work; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 4377, "end_line": 4546, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4377-L4546", "decorators": [], "calls": [ "_time.monotonic", "logger.info", "generator.__aiter__", "asyncio.create_task", "_wait_disconnect", "logger.warning", "_elapsed", "anext_task.done", "anext_task.cancel", "asyncio.ensure_future", "aiter.__anext__", "asyncio.wait", "min", "anext_task.result", "logger.error", "type", "disconnect_task.done", "disconnect_task.cancel", "_deferred_generator_close", "cleanup", "asyncio.iscoroutine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": true }, { "name": "_elapsed", "qualname": "_disconnect_guard._elapsed", "full_name": "vllm_mlx.server._disconnect_guard._elapsed", "kind": "nested function", "signature": "def _elapsed()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_disconnect_guard._elapsed` calls `_time.monotonic`; returns `f'{_time.monotonic() - _t0:.1f}s'`.", "implementation": "Nested Function `_disconnect_guard._elapsed` calls `_time.monotonic`; returns `f'{_time.monotonic() - _t0:.1f}s'`.", "documented": false, "public": false, "addressable": false, "line": 4407, "end_line": 4408, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4407-L4408", "decorators": [], "calls": [ "_time.monotonic" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'{_time.monotonic() - _t0:.1f}s'" ], "awaits": false, "yields": false }, { "name": "_wait_disconnect", "qualname": "_disconnect_guard._wait_disconnect", "full_name": "vllm_mlx.server._disconnect_guard._wait_disconnect", "kind": "nested function", "signature": "async def _wait_disconnect()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_disconnect_guard._wait_disconnect` calls `asyncio.sleep`, `_is_client_disconnected`, `logger.info`, `_elapsed`; awaits asynchronous work; returns `None`.", "implementation": "Nested Function `_disconnect_guard._wait_disconnect` calls `asyncio.sleep`, `_is_client_disconnected`, `logger.info`, `_elapsed`; awaits asynchronous work; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 4417, "end_line": 4429, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4417-L4429", "decorators": [], "calls": [ "asyncio.sleep", "_is_client_disconnected", "logger.info", "_elapsed" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_deferred_generator_close", "qualname": "_disconnect_guard._deferred_generator_close", "full_name": "vllm_mlx.server._disconnect_guard._deferred_generator_close", "kind": "nested function", "signature": "async def _deferred_generator_close()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_disconnect_guard._deferred_generator_close` calls `asyncio.sleep`, `_gen_to_close.aclose`, `logger.debug`, `type`; awaits asynchronous work.", "implementation": "Nested Function `_disconnect_guard._deferred_generator_close` calls `asyncio.sleep`, `_gen_to_close.aclose`, `logger.debug`, `type`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 4528, "end_line": 4536, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4528-L4536", "decorators": [], "calls": [ "asyncio.sleep", "_gen_to_close.aclose", "logger.debug", "type" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_wait_with_disconnect", "qualname": "_wait_with_disconnect", "full_name": "vllm_mlx.server._wait_with_disconnect", "kind": "function", "signature": "async def _wait_with_disconnect(coro, raw_request: Request, timeout: float, poll_interval: float=0.5, timeout_detail_seconds: float | None=None, cleanup_result=None)", "parameters": [ { "name": "coro", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "timeout", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "poll_interval", "kind": "positional or keyword", "annotation": "float", "default": "0.5", "required": false, "description": "Optional positional or keyword input; defaults to `0.5`." }, { "name": "timeout_detail_seconds", "kind": "positional or keyword", "annotation": "float | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "cleanup_result", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Run a coroutine with both timeout and client disconnect detection.\n\nFor non-streaming requests where _disconnect_guard() can't be used.\nRaces the coroutine against a disconnect poller, same pattern as\n_disconnect_guard but for awaitable (non-generator) coroutines.", "summary": "Run a coroutine with both timeout and client disconnect detection.", "implementation": "Function `_wait_with_disconnect` calls `_time.monotonic`, `asyncio.ensure_future`, `asyncio.create_task`, `_wait_disconnect`; awaits asynchronous work; can raise `HTTPException`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 4549, "end_line": 4638, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4549-L4638", "decorators": [], "calls": [ "_time.monotonic", "asyncio.ensure_future", "asyncio.create_task", "_wait_disconnect", "asyncio.wait", "task.cancel", "HTTPException", "logger.info", "task.result", "cleanup_result", "asyncio.iscoroutine", "disconnect_task.done", "disconnect_task.cancel", "task.done" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "None", "task.result()" ], "awaits": true, "yields": false }, { "name": "_wait_disconnect", "qualname": "_wait_with_disconnect._wait_disconnect", "full_name": "vllm_mlx.server._wait_with_disconnect._wait_disconnect", "kind": "nested function", "signature": "async def _wait_disconnect()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_wait_with_disconnect._wait_disconnect` calls `asyncio.sleep`, `_is_client_disconnected`, `logger.info`, `_time.monotonic`; awaits asynchronous work; returns `None`.", "implementation": "Nested Function `_wait_with_disconnect._wait_disconnect` calls `asyncio.sleep`, `_is_client_disconnected`, `logger.info`, `_time.monotonic`; awaits asynchronous work; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 4569, "end_line": 4581, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4569-L4581", "decorators": [], "calls": [ "asyncio.sleep", "_is_client_disconnected", "logger.info", "_time.monotonic" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_start_request_budget", "qualname": "_start_request_budget", "full_name": "vllm_mlx.server._start_request_budget", "kind": "function", "signature": "def _start_request_budget(timeout: float | None) -> tuple[float, float]", "parameters": [ { "name": "timeout", "kind": "positional or keyword", "annotation": "float | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[float, float]", "docstring": "Return the total timeout and absolute deadline for a request.", "summary": "Return the total timeout and absolute deadline for a request.", "implementation": "Function `_start_request_budget` calls `time.monotonic`; returns `(total_timeout, time.monotonic() + total_timeout)`.", "documented": true, "public": false, "addressable": true, "line": 4641, "end_line": 4644, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4641-L4644", "decorators": [], "calls": [ "time.monotonic" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(total_timeout, time.monotonic() + total_timeout)" ], "awaits": false, "yields": false }, { "name": "_remaining_request_timeout", "qualname": "_remaining_request_timeout", "full_name": "vllm_mlx.server._remaining_request_timeout", "kind": "function", "signature": "def _remaining_request_timeout(total_timeout: float, deadline: float) -> float", "parameters": [ { "name": "total_timeout", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "deadline", "kind": "positional or keyword", "annotation": "float", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "float", "docstring": "Compute remaining request budget or raise the standard timeout error.", "summary": "Compute remaining request budget or raise the standard timeout error.", "implementation": "Function `_remaining_request_timeout` calls `time.monotonic`, `HTTPException`; can raise `HTTPException`; returns `remaining`.", "documented": true, "public": false, "addressable": true, "line": 4647, "end_line": 4655, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4647-L4655", "decorators": [], "calls": [ "time.monotonic", "HTTPException" ], "state_reads": [], "state_writes": [], "raises": [ "HTTPException" ], "return_expressions": [ "remaining" ], "awaits": false, "yields": false }, { "name": "_acquire_default_engine_for_request", "qualname": "_acquire_default_engine_for_request", "full_name": "vllm_mlx.server._acquire_default_engine_for_request", "kind": "function", "signature": "async def _acquire_default_engine_for_request(raw_request: Request, *, total_timeout: float, deadline: float, count_activity: bool=True, model: str | None=None) -> BaseEngine | None", "parameters": [ { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "total_timeout", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "deadline", "kind": "keyword-only", "annotation": "float", "default": "", "required": true, "description": "Required keyword-only input." }, { "name": "count_activity", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." }, { "name": "model", "kind": "keyword-only", "annotation": "str | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." } ], "return_annotation": "BaseEngine | None", "docstring": "Acquire the engine for a request, using the model registry when active.\n\nWhen ``_model_manager`` is set (registry mode), acquires the engine for the\nrequested *model* via ``_acquire_request_model``. The resulting\n``RequestModelContext`` is stashed in ``_active_request_contexts`` keyed by\n``id(raw_request)`` so that the matching ``_release_default_engine`` call\ncan release the lease.\n\nIn single-model mode the behaviour is unchanged.", "summary": "Acquire the engine for a request, using the model registry when active.", "implementation": "Function `_acquire_default_engine_for_request` calls `_registry_acquire`, `_wait_with_disconnect`, `_remaining_request_timeout`, `_acquire_default_engine`; awaits asynchronous work; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 4661, "end_line": 4719, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4661-L4719", "decorators": [], "calls": [ "_registry_acquire", "_wait_with_disconnect", "_remaining_request_timeout", "_acquire_default_engine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "await _registry_acquire()", "await _wait_with_disconnect(_registry_acquire(), raw_request, timeout=_remaining_request_timeout(total_timeout, deadlin…", "await acquire_coro", "await _wait_with_disconnect(acquire_coro, raw_request, timeout=_remaining_request_timeout(total_timeout, deadline), tim…" ], "awaits": true, "yields": false }, { "name": "_registry_acquire", "qualname": "_acquire_default_engine_for_request._registry_acquire", "full_name": "vllm_mlx.server._acquire_default_engine_for_request._registry_acquire", "kind": "nested function", "signature": "async def _registry_acquire()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_acquire_default_engine_for_request._registry_acquire` calls `_acquire_request_model`, `id`; awaits asynchronous work; returns `ctx.engine`.", "implementation": "Nested Function `_acquire_default_engine_for_request._registry_acquire` calls `_acquire_request_model`, `id`; awaits asynchronous work; returns `ctx.engine`.", "documented": false, "public": false, "addressable": false, "line": 4681, "end_line": 4685, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4681-L4685", "decorators": [], "calls": [ "_acquire_request_model", "id" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "ctx.engine" ], "awaits": true, "yields": false }, { "name": "_registry_cleanup", "qualname": "_acquire_default_engine_for_request._registry_cleanup", "full_name": "vllm_mlx.server._acquire_default_engine_for_request._registry_cleanup", "kind": "nested function", "signature": "async def _registry_cleanup(_result)", "parameters": [ { "name": "_result", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_acquire_default_engine_for_request._registry_cleanup` calls `_active_request_contexts.pop`, `id`, `ctx.release`; awaits asynchronous work.", "implementation": "Nested Function `_acquire_default_engine_for_request._registry_cleanup` calls `_active_request_contexts.pop`, `id`, `ctx.release`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 4687, "end_line": 4690, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4687-L4690", "decorators": [], "calls": [ "_active_request_contexts.pop", "id", "ctx.release" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_release_engine_for_request", "qualname": "_release_engine_for_request", "full_name": "vllm_mlx.server._release_engine_for_request", "kind": "function", "signature": "async def _release_engine_for_request(raw_request: Request | None, *, count_activity: bool=True) -> None", "parameters": [ { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "count_activity", "kind": "keyword-only", "annotation": "bool", "default": "True", "required": false, "description": "Optional keyword-only input; defaults to `True`." } ], "return_annotation": "None", "docstring": "Release the engine acquired for this request.\n\nIn registry mode, releases the model lease stashed by\n``_acquire_default_engine_for_request``. In single-model mode, falls\nthrough to the default release path. ``count_activity`` must match the\nflag used on the matching acquire so idle-unload accounting stays correct.", "summary": "Release the engine acquired for this request.", "implementation": "Function `_release_engine_for_request` calls `_active_request_contexts.pop`, `id`, `ctx.release`, `_release_default_engine`; awaits asynchronous work; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 4722, "end_line": 4737, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4722-L4737", "decorators": [], "calls": [ "_active_request_contexts.pop", "id", "ctx.release", "_release_default_engine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": true, "yields": false }, { "name": "_make_release_cleanup", "qualname": "_make_release_cleanup", "full_name": "vllm_mlx.server._make_release_cleanup", "kind": "function", "signature": "def _make_release_cleanup(raw_request: Request | None)", "parameters": [ { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Return a cleanup callable suitable for ``_disconnect_guard``.", "summary": "Return a cleanup callable suitable for ``_disconnect_guard``.", "implementation": "Function `_make_release_cleanup` has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 4740, "end_line": 4752, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4740-L4752", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_cleanup", "_release_default_engine" ], "awaits": false, "yields": false }, { "name": "_cleanup", "qualname": "_make_release_cleanup._cleanup", "full_name": "vllm_mlx.server._make_release_cleanup._cleanup", "kind": "nested function", "signature": "async def _cleanup()", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_make_release_cleanup._cleanup` calls `_active_request_contexts.pop`, `id`, `ctx.release`, `_release_default_engine`; awaits asynchronous work.", "implementation": "Nested Function `_make_release_cleanup._cleanup` calls `_active_request_contexts.pop`, `id`, `ctx.release`, `_release_default_engine`; awaits asynchronous work.", "documented": false, "public": false, "addressable": false, "line": 4744, "end_line": 4749, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4744-L4749", "decorators": [], "calls": [ "_active_request_contexts.pop", "id", "ctx.release", "_release_default_engine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "create_completion", "qualname": "create_completion", "full_name": "vllm_mlx.server.create_completion", "kind": "function", "signature": "async def create_completion(request: CompletionRequest, raw_request: Request)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "CompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Create a text completion.", "summary": "Create a text completion.", "implementation": "Function `create_completion` calls `_validate_model_name`, `_resolve_request_max_tokens`, `_metrics.track_inference`, `isinstance`; awaits asynchronous work; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 4763, "end_line": 4909, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4763-L4909", "decorators": [ "app.post('/v1/completions', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "_validate_model_name", "_resolve_request_max_tokens", "_metrics.track_inference", "isinstance", "_start_request_budget", "sum", "len", "logger.info", "_sanitize_log_text", "_acquire_default_engine_for_request", "Response", "StreamingResponse", "_disconnect_guard", "_ensure_sse_terminal", "stream_completion", "_make_release_cleanup", "time.perf_counter", "enumerate", "_resolve_temperature", "_resolve_top_p", "_resolve_top_k", "_resolve_min_p", "_resolve_presence_penalty", "_resolve_repetition_penalty", "getattr", "engine.generate", "_wait_with_disconnect", "_remaining_request_timeout", "tracker.finish", "_metrics_result_from_status", "_raise_engine_busy", "choices.append", "CompletionChoice", "hasattr", "CompletionResponse", "_response_model_name", "Usage", "_release_engine_for_request" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Response(status_code=499)", "response", "CompletionResponse(model=_response_model_name(request.model), choices=choices, usage=Usage(prompt_tokens=total_prompt_t…" ], "awaits": true, "yields": false }, { "name": "create_chat_completion", "qualname": "create_chat_completion", "full_name": "vllm_mlx.server.create_chat_completion", "kind": "function", "signature": "async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Create a chat completion (supports multimodal content for VLM models).\n\nOpenAI-compatible multimodal format for images:\n```json\nmessages=[{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"text\", \"text\": \"What's in this image?\"},\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://...\"}}\n ]\n}]\n```\n\nVideo support:\n```json\nmessages=[{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"text\", \"text\": \"What happens in this video?\"},\n {\"type\": \"video_url\", \"video_url\": {\"url\": \"https://example.com/video.mp4\"}}\n ]\n}]\n```\n\nStructured output (JSON mode):\n```json\nresponse_format={\"type\": \"json_object\"}\n```\n\nStructured output (JSON Schema):\n```json\nresponse_format={\n \"type\": \"json_schema\",\n \"json_schema\": {\n \"name\": \"my_schema\",\n \"schema\": {\"type\": \"object\", \"properties\": {...}}\n }\n}\n```", "summary": "Create a chat completion (supports multimodal content for VLM models).", "implementation": "Function `create_chat_completion` calls `_validate_model_name`, `_resolve_request_max_tokens`, `_metrics.track_inference`, `_start_request_budget`; awaits asynchronous work; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 4916, "end_line": 5114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L4916-L5114", "decorators": [ "app.post('/v1/chat/completions', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "_validate_model_name", "_resolve_request_max_tokens", "_metrics.track_inference", "_start_request_budget", "len", "isinstance", "str", "logger.info", "_sanitize_log_text", "_acquire_default_engine_for_request", "Response", "_prepare_chat_completion_invocation", "tracker.finish", "_raise_remote_media_http_error", "StreamingResponse", "_disconnect_guard", "_ensure_sse_terminal", "stream_chat_completion", "_make_release_cleanup", "time.perf_counter", "_wait_with_disconnect", "engine.chat", "_remaining_request_timeout", "_metrics_result_from_status", "_raise_engine_busy", "_extract_reasoning_and_tool_calls", "_thinking_disabled", "_apply_response_format_or_raise", "logger.error", "logger.warning", "ChatCompletionResponse", "_response_model_name", "ChatCompletionChoice", "AssistantMessage", "clean_output_text", "Usage", "_generation_metadata", "_release_engine_for_request" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Response(status_code=499)", "response", "ChatCompletionResponse(model=_response_model_name(request.model), choices=[ChatCompletionChoice(message=AssistantMessag…" ], "awaits": true, "yields": false }, { "name": "_normalize_messages", "qualname": "_normalize_messages", "full_name": "vllm_mlx.server._normalize_messages", "kind": "function", "signature": "def _normalize_messages(messages: list[dict]) -> list[dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "List of message dicts with 'role' and 'content' keys." } ], "return_annotation": "list[dict]", "docstring": "Normalize message roles and merge consecutive same-role messages.\n\n1. Maps non-standard roles to standard ones (e.g. ``developer`` -> ``system``).\n2. Merges consecutive same-role messages to satisfy chat template constraints\n (Qwen 3.5, Llama, etc. require alternating roles).\n\nOnly merges when both messages have string content. Messages with list\ncontent (multimodal) are left as-is to preserve image/video attachments.\n\nArgs:\n messages: List of message dicts with 'role' and 'content' keys.\n\nReturns:\n New list with normalized roles and consecutive same-role messages merged.", "summary": "Normalize message roles and merge consecutive same-role messages.", "implementation": "Function `_normalize_messages` calls `messages[0].copy`, `_ROLE_MAP.get`, `isinstance`, `prev.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 5117, "end_line": 5172, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5117-L5172", "decorators": [], "calls": [ "messages[0].copy", "_ROLE_MAP.get", "isinstance", "prev.get", "msg.get", "logger.debug", "len", "msg.copy", "merged.append", "sum", "parts.append", "logger.info", "', '.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "messages", "merged" ], "awaits": false, "yields": false }, { "name": "_get_engine_tokenizer", "qualname": "_get_engine_tokenizer", "full_name": "vllm_mlx.server._get_engine_tokenizer", "kind": "function", "signature": "def _get_engine_tokenizer(engine) -> object | None", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "object | None", "docstring": "Return the tokenizer backing ``engine``, if exposed.\n\nDifferent engine classes store the tokenizer under different attributes.\nWe try the common ones and return ``None`` if nothing matches, so that\noptional features like constrained decoding can degrade gracefully.", "summary": "Return the tokenizer backing ``engine``, if exposed.", "implementation": "Function `_get_engine_tokenizer` calls `getattr`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 5175, "end_line": 5187, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5175-L5187", "decorators": [], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tok", "None" ], "awaits": false, "yields": false }, { "name": "create_response", "qualname": "create_response", "full_name": "vllm_mlx.server.create_response", "kind": "function", "signature": "async def create_response(request: ResponsesRequest, raw_request: Request)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "ResponsesRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "raw_request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Create a Responses API response.", "summary": "Create a Responses API response.", "implementation": "Function `create_response` calls `_responses_request_to_chat_request`, `_validate_remote_media_urls`, `StreamingResponse`, `_disconnect_guard`; awaits asynchronous work; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 5194, "end_line": 5214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5194-L5214", "decorators": [ "app.post('/v1/responses', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "_responses_request_to_chat_request", "_validate_remote_media_urls", "StreamingResponse", "_disconnect_guard", "_stream_responses_request", "_run_responses_request", "_raise_remote_media_http_error", "Response" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "StreamingResponse(_disconnect_guard(_stream_responses_request(request), raw_request), media_type='text/event-stream')", "Response(status_code=499)", "response_object" ], "awaits": true, "yields": false }, { "name": "_get_forced_tool_name", "qualname": "_get_forced_tool_name", "full_name": "vllm_mlx.server._get_forced_tool_name", "kind": "function", "signature": "def _get_forced_tool_name(tool_choice) -> str | None", "parameters": [ { "name": "tool_choice", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "Extract forced tool name from tool_choice, if any.\n\nReturns the function name when tool_choice is a dict like\n{\"type\": \"function\", \"function\": {\"name\": \"X\"}}, or None otherwise.", "summary": "Extract forced tool name from tool_choice, if any.", "implementation": "Function `_get_forced_tool_name` calls `isinstance`, `tool_choice.get`, `func.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 5217, "end_line": 5230, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5217-L5230", "decorators": [], "calls": [ "isinstance", "tool_choice.get", "func.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "func.get('name')" ], "awaits": false, "yields": false }, { "name": "_apply_forced_tool_choice", "qualname": "_apply_forced_tool_choice", "full_name": "vllm_mlx.server._apply_forced_tool_choice", "kind": "function", "signature": "def _apply_forced_tool_choice(tool_choice, tools, messages, chat_kwargs=None)", "parameters": [ { "name": "tool_choice", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "The tool_choice value from the request" }, { "name": "tools", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "List of converted tools for the template" }, { "name": "messages", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "The message list (will be copied if modified)" }, { "name": "chat_kwargs", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional dict to modify (e.g. disable thinking)" } ], "return_annotation": "not annotated", "docstring": "Apply forced tool_choice by filtering tools and injecting instructions.\n\nHandles:\n- tool_choice={\"type\":\"function\",\"function\":{\"name\":\"X\"}} -> filter + instruct\n- tool_choice=\"required\" -> instruct model to call at least one tool\n\nArgs:\n tool_choice: The tool_choice value from the request\n tools: List of converted tools for the template\n messages: The message list (will be copied if modified)\n chat_kwargs: Optional dict to modify (e.g. disable thinking)\n\nReturns:\n Tuple of (tools, messages) - potentially filtered/modified", "summary": "Apply forced tool_choice by filtering tools and injecting instructions.", "implementation": "Function `_apply_forced_tool_choice` calls `_get_forced_tool_name`, `_tool_name`, `ValueError`, `_inject_json_instruction`; can raise `ValueError`; returns `(tools, messages)`.", "documented": true, "public": false, "addressable": true, "line": 5233, "end_line": 5279, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5233-L5279", "decorators": [], "calls": [ "_get_forced_tool_name", "_tool_name", "ValueError", "_inject_json_instruction" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "(tools, messages)" ], "awaits": false, "yields": false }, { "name": "_tool_name", "qualname": "_tool_name", "full_name": "vllm_mlx.server._tool_name", "kind": "function", "signature": "def _tool_name(tool: dict) -> str | None", "parameters": [ { "name": "tool", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "Extract function name from a tool definition dict.", "summary": "Extract function name from a tool definition dict.", "implementation": "Function `_tool_name` calls `tool.get`, `isinstance`, `func.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 5282, "end_line": 5287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5282-L5287", "decorators": [], "calls": [ "tool.get", "isinstance", "func.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "func.get('name')", "None" ], "awaits": false, "yields": false }, { "name": "_inject_json_instruction", "qualname": "_inject_json_instruction", "full_name": "vllm_mlx.server._inject_json_instruction", "kind": "function", "signature": "def _inject_json_instruction(messages: list, instruction: str) -> list", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "instruction", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list", "docstring": "Inject JSON instruction into messages.\n\nIf a system message exists, append to it. Otherwise, prepend a new system message.", "summary": "Inject JSON instruction into messages.", "implementation": "Function `_inject_json_instruction` calls `list`, `enumerate`, `isinstance`, `msg.get`; returns `messages`.", "documented": true, "public": false, "addressable": true, "line": 5290, "end_line": 5319, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5290-L5319", "decorators": [], "calls": [ "list", "enumerate", "isinstance", "msg.get", "getattr", "messages.insert" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "messages" ], "awaits": false, "yields": false }, { "name": "_convert_anthropic_stop_reason", "qualname": "_convert_anthropic_stop_reason", "full_name": "vllm_mlx.server._convert_anthropic_stop_reason", "kind": "function", "signature": "def _convert_anthropic_stop_reason(openai_reason: str | None) -> str", "parameters": [ { "name": "openai_reason", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Convert OpenAI finish_reason to Anthropic stop_reason.", "summary": "Convert OpenAI finish_reason to Anthropic stop_reason.", "implementation": "Function `_convert_anthropic_stop_reason` calls `mapping.get`; returns `mapping.get(openai_reason or '', 'end_turn')`.", "documented": true, "public": false, "addressable": true, "line": 5327, "end_line": 5335, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5327-L5335", "decorators": [], "calls": [ "mapping.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mapping.get(openai_reason or '', 'end_turn')" ], "awaits": false, "yields": false }, { "name": "_prepare_anthropic_endpoint_invocation", "qualname": "_prepare_anthropic_endpoint_invocation", "full_name": "vllm_mlx.server._prepare_anthropic_endpoint_invocation", "kind": "function", "signature": "def _prepare_anthropic_endpoint_invocation(engine: BaseEngine, openai_request: ChatCompletionRequest, effective_max_tokens: int) -> PreparedChatInvocation", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "openai_request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "effective_max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "PreparedChatInvocation", "docstring": "Prepare Anthropic invocation and convert URL-safety errors to 400s.", "summary": "Prepare Anthropic invocation and convert URL-safety errors to 400s.", "implementation": "Function `_prepare_anthropic_endpoint_invocation` calls `_prepare_anthropic_invocation`, `_raise_remote_media_http_error`; returns `_prepare_anthropic_invocation(engine, openai_request, effective_max_tokens)`.", "documented": true, "public": false, "addressable": true, "line": 5338, "end_line": 5351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5338-L5351", "decorators": [], "calls": [ "_prepare_anthropic_invocation", "_raise_remote_media_http_error" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_prepare_anthropic_invocation(engine, openai_request, effective_max_tokens)" ], "awaits": false, "yields": false }, { "name": "create_anthropic_message", "qualname": "create_anthropic_message", "full_name": "vllm_mlx.server.create_anthropic_message", "kind": "function", "signature": "async def create_anthropic_message(request: Request)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Anthropic Messages API endpoint.\n\nTranslates Anthropic-format requests to OpenAI format, runs inference\nthrough the existing engine, and converts the response back.\n\nSupports both streaming and non-streaming modes.", "summary": "Anthropic Messages API endpoint.", "implementation": "Function `create_anthropic_message` calls `_metrics.track_inference`, `request.json`, `str`, `request.body`; awaits asynchronous work; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 5357, "end_line": 5578, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5357-L5578", "decorators": [ "app.post('/v1/messages', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "_metrics.track_inference", "request.json", "str", "request.body", "json.loads", "re.sub", "AnthropicRequest", "_validate_model_name", "_resolve_request_max_tokens", "len", "isinstance", "logger.info", "_sanitize_log_text", "anthropic_to_openai", "_start_request_budget", "_acquire_default_engine_for_request", "Response", "_prepare_anthropic_endpoint_invocation", "json.dumps", "StreamingResponse", "_disconnect_guard", "_ensure_sse_terminal", "_stream_anthropic_messages", "_make_release_cleanup", "time.perf_counter", "_wait_with_disconnect", "engine.chat", "_remaining_request_timeout", "tracker.finish", "_metrics_result_from_status", "_extract_reasoning_and_tool_calls", "_thinking_disabled", "_apply_response_format_or_raise", "logger.error", "logger.warning", "clean_output_text", "content_blocks.append", "AnthropicResponseContentBlock", "_convert_anthropic_stop_reason", "AnthropicResponse", "_response_model_name", "AnthropicUsage", "anthropic_response.model_dump_json", "_release_engine_for_request" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Response(status_code=499)", "response", "Response(content=anthropic_response.model_dump_json(exclude_none=True), media_type='application/json')" ], "awaits": true, "yields": false }, { "name": "count_anthropic_tokens", "qualname": "count_anthropic_tokens", "full_name": "vllm_mlx.server.count_anthropic_tokens", "kind": "function", "signature": "async def count_anthropic_tokens(request: Request)", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "Request", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Count tokens for an Anthropic Messages API request.\n\nUses the model's tokenizer for accurate counting.\nClaude Code calls this endpoint for token budgeting.\nNote: Don't parse via AnthropicRequest — count_tokens requests\nfrom Claude Code don't include max_tokens.", "summary": "Count tokens for an Anthropic Messages API request.", "implementation": "Function `count_anthropic_tokens` calls `request.json`, `body.get`, `isinstance`, `_validate_model_name`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 5585, "end_line": 5666, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5585-L5666", "decorators": [ "app.post('/v1/messages/count_tokens', dependencies=[Depends(verify_api_key), Depends(check_rate_limit)])" ], "calls": [ "request.json", "body.get", "isinstance", "_validate_model_name", "_start_request_budget", "_acquire_default_engine_for_request", "Response", "len", "tokenizer.encode", "block.get", "msg.get", "json.dumps", "item.get", "tool.get", "_release_engine_for_request" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "Response(status_code=499)", "{'input_tokens': total_tokens}" ], "awaits": true, "yields": false }, { "name": "_emit_content_pieces", "qualname": "_emit_content_pieces", "full_name": "vllm_mlx.server._emit_content_pieces", "kind": "function", "signature": "def _emit_content_pieces(pieces: list[tuple[str, str]], current_block_type: str | None, block_index: int) -> tuple[list[str], str | None, int]", "parameters": [ { "name": "pieces", "kind": "positional or keyword", "annotation": "list[tuple[str, str]]", "default": "", "required": true, "description": "List of (block_type, text) from StreamingThinkRouter" }, { "name": "current_block_type", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Current open block type, or None" }, { "name": "block_index", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Current block index" } ], "return_annotation": "tuple[list[str], str | None, int]", "docstring": "Emit Anthropic SSE events for content pieces from the think router.\n\nHandles block type transitions (thinking <-> text), emitting\ncontent_block_start/stop/delta events as needed.\n\nArgs:\n pieces: List of (block_type, text) from StreamingThinkRouter\n current_block_type: Current open block type, or None\n block_index: Current block index\n\nReturns:\n Tuple of (events, updated_block_type, updated_block_index)", "summary": "Emit Anthropic SSE events for content pieces from the think router.", "implementation": "Function `_emit_content_pieces` calls `events.append`, `json.dumps`; returns `(events, current_block_type, block_index)`.", "documented": true, "public": false, "addressable": true, "line": 5669, "end_line": 5719, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5669-L5719", "decorators": [], "calls": [ "events.append", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(events, current_block_type, block_index)" ], "awaits": false, "yields": false }, { "name": "_stream_anthropic_messages", "qualname": "_stream_anthropic_messages", "full_name": "vllm_mlx.server._stream_anthropic_messages", "kind": "function", "signature": "async def _stream_anthropic_messages(engine: BaseEngine, openai_request: ChatCompletionRequest, anthropic_request: AnthropicRequest, prepared: PreparedChatInvocation, metrics_tracker=None) -> AsyncIterator[str]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "openai_request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "anthropic_request", "kind": "positional or keyword", "annotation": "AnthropicRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prepared", "kind": "positional or keyword", "annotation": "PreparedChatInvocation", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "metrics_tracker", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "AsyncIterator[str]", "docstring": "Stream Anthropic Messages API SSE events.\n\nConverts OpenAI streaming chunks to Anthropic event format:\nmessage_start -> content_block_start -> content_block_delta* ->\ncontent_block_stop -> message_delta -> message_stop\n\nWhen a reasoning parser is active, emits a ``thinking`` content block\n(index 0) for reasoning tokens and a ``text`` content block (index 1)\nfor the actual response, matching the Anthropic extended thinking format.", "summary": "Stream Anthropic Messages API SSE events.", "implementation": "Function `_stream_anthropic_messages` calls `uuid.uuid4`, `time.perf_counter`, `dict`, `_response_model_name`; yields values incrementally.", "documented": true, "public": false, "addressable": true, "line": 5722, "end_line": 5995, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L5722-L5995", "decorators": [], "calls": [ "uuid.uuid4", "time.perf_counter", "dict", "_response_model_name", "json.dumps", "_prepare_streaming_reasoning_parser", "chat_kwargs.get", "_get_streaming_tool_parser", "openai_request.model_dump", "engine.stream_chat", "metrics_tracker.observe_ttft", "hasattr", "SPECIAL_TOKENS_PATTERN.sub", "_streaming_tool_markup_possible_after_delta", "_parse_streaming_tool_content", "tool_result.get", "_TOOL_MARKUP_PATTERN.sub", "reasoning_parser.extract_reasoning_streaming", "_parse_tool_calls_with_parser", "enumerate", "json.loads", "logger.info", "_metrics_result_from_status", "metrics_tracker.finish" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "stream_completion", "qualname": "stream_completion", "full_name": "vllm_mlx.server.stream_completion", "kind": "function", "signature": "async def stream_completion(engine: BaseEngine, prompt: str, request: CompletionRequest, max_tokens: int, repetition_penalty: float | None=None, metrics_tracker=None) -> AsyncIterator[str]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "CompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "max_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "repetition_penalty", "kind": "positional or keyword", "annotation": "float | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "metrics_tracker", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "AsyncIterator[str]", "docstring": "Stream completion response.", "summary": "Stream completion response.", "implementation": "Function `stream_completion` calls `_resolve_temperature`, `_resolve_top_p`, `_resolve_top_k`, `_resolve_min_p`; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 6003, "end_line": 6084, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6003-L6084", "decorators": [], "calls": [ "_resolve_temperature", "_resolve_top_p", "_resolve_top_k", "_resolve_min_p", "_resolve_presence_penalty", "_resolve_repetition_penalty", "getattr", "engine.stream_generate", "metrics_tracker.observe_ttft", "hasattr", "uuid.uuid4", "int", "time.time", "_response_model_name", "get_usage(output).model_dump", "get_usage", "json.dumps", "_metrics_result_from_status", "metrics_tracker.finish" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "stream_chat_completion", "qualname": "stream_chat_completion", "full_name": "vllm_mlx.server.stream_chat_completion", "kind": "function", "signature": "async def stream_chat_completion(engine: BaseEngine, messages: list, request: ChatCompletionRequest, metrics_tracker=None, **kwargs) -> AsyncIterator[str]", "parameters": [ { "name": "engine", "kind": "positional or keyword", "annotation": "BaseEngine", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "messages", "kind": "positional or keyword", "annotation": "list", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "ChatCompletionRequest", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "metrics_tracker", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "**kwargs", "kind": "variadic keyword", "annotation": "not annotated", "default": "", "required": false, "description": "Additional variadic keyword inputs accepted by this callable." } ], "return_annotation": "AsyncIterator[str]", "docstring": "Stream chat completion response.", "summary": "Stream chat completion response.", "implementation": "Function `stream_chat_completion` calls `uuid.uuid4`, `time.perf_counter`, `_stream_request_metadata`, `ChatCompletionChunk`; yields values incrementally.", "documented": true, "public": true, "addressable": true, "line": 6087, "end_line": 6512, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6087-L6512", "decorators": [], "calls": [ "uuid.uuid4", "time.perf_counter", "_stream_request_metadata", "ChatCompletionChunk", "_response_model_name", "ChatCompletionChunkChoice", "ChatCompletionChunkDelta", "first_chunk.model_dump_json", "_prepare_openai_stream_reasoning_state", "_streaming_json_fence_stripper", "_get_streaming_tool_parser", "engine.stream_chat", "metrics_tracker.observe_ttft", "hasattr", "reasoning_parser.extract_reasoning_streaming", "_promote_streaming_response_format_delta", "_streaming_tool_markup_possible", "_streaming_tool_markup_possible_after_delta", "_extract_streaming_tool_delta", "chunk.model_dump_json", "tc.get", "_coerce_tool_arguments", "get_usage", "tool_result.get", "_TOOL_MARKUP_PATTERN.sub", "fence_stripper.feed", "fence_stripper.finalize", "SPECIAL_TOKENS_PATTERN.sub", "tool_parser.extract_tool_calls", "enumerate", "tool_chunk.model_dump_json", "getattr", "parse_json_output", "any", "kwargs.get", "logger.error", "logger.warning", "logger.info", "Usage", "usage_chunk.model_dump_json", "_metrics_result_from_status", "metrics_tracker.finish" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": true }, { "name": "init_mcp", "qualname": "init_mcp", "full_name": "vllm_mlx.server.init_mcp", "kind": "function", "signature": "async def init_mcp(config_path: str)", "parameters": [ { "name": "config_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Initialize MCP manager from config file.", "summary": "Initialize MCP manager from config file.", "implementation": "Function `init_mcp` calls `load_mcp_config`, `MCPClientManager`, `_mcp_manager.start`, `ToolSandbox`; awaits asynchronous work.", "documented": true, "public": true, "addressable": true, "line": 6520, "end_line": 6546, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6520-L6546", "decorators": [], "calls": [ "load_mcp_config", "MCPClientManager", "_mcp_manager.start", "ToolSandbox", "ToolExecutor", "logger.info", "len", "_mcp_manager.get_all_tools", "logger.error", "_sanitize_log_text" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": true, "yields": false }, { "name": "_make_keepalive_http_protocol", "qualname": "_make_keepalive_http_protocol", "full_name": "vllm_mlx.server._make_keepalive_http_protocol", "kind": "function", "signature": "def _make_keepalive_http_protocol(idle=10, interval=5, count=3)", "parameters": [ { "name": "idle", "kind": "positional or keyword", "annotation": "not annotated", "default": "10", "required": false, "description": "Optional positional or keyword input; defaults to `10`." }, { "name": "interval", "kind": "positional or keyword", "annotation": "not annotated", "default": "5", "required": false, "description": "Optional positional or keyword input; defaults to `5`." }, { "name": "count", "kind": "positional or keyword", "annotation": "not annotated", "default": "3", "required": false, "description": "Optional positional or keyword input; defaults to `3`." } ], "return_annotation": "not annotated", "docstring": "Create a uvicorn HTTP protocol class with aggressive TCP keepalive.\n\nWhen a client abruptly disconnects (power-off, network loss), the server\nTCP stack won't notice for ~2 hours (default keepalive). With aggressive\nkeepalive (idle=10s, interval=5s, count=3), dead connections are detected\nin ~25 seconds, letting ``_wait_with_disconnect()`` abort the request and\nstop wasting GPU cycles on tokens nobody will receive.", "summary": "Create a uvicorn HTTP protocol class with aggressive TCP keepalive.", "implementation": "Function `_make_keepalive_http_protocol` returns `_KeepaliveProtocol`.", "documented": true, "public": false, "addressable": true, "line": 6554, "end_line": 6589, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6554-L6589", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_KeepaliveProtocol" ], "awaits": false, "yields": false }, { "name": "_KeepaliveProtocol", "qualname": "_make_keepalive_http_protocol._KeepaliveProtocol", "full_name": "vllm_mlx.server._make_keepalive_http_protocol._KeepaliveProtocol", "kind": "nested class", "signature": "class _KeepaliveProtocol(_Base)", "parameters": [], "return_annotation": "_KeepaliveProtocol", "docstring": "", "summary": "Nested Class `_make_keepalive_http_protocol._KeepaliveProtocol` derives from `_Base` and declares 1 direct member(s).", "implementation": "Nested Class `_make_keepalive_http_protocol._KeepaliveProtocol` derives from `_Base` and declares 1 direct member(s).", "documented": false, "public": false, "addressable": false, "line": 6567, "end_line": 6587, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6567-L6587", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "connection_made", "qualname": "_make_keepalive_http_protocol._KeepaliveProtocol.connection_made", "full_name": "vllm_mlx.server._make_keepalive_http_protocol._KeepaliveProtocol.connection_made", "kind": "nested function", "signature": "def connection_made(self, transport)", "parameters": [ { "name": "transport", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `_make_keepalive_http_protocol._KeepaliveProtocol.connection_made` calls `super().connection_made`, `super`, `transport.get_extra_info`, `sock.setsockopt`; returns `None`.", "implementation": "Nested Function `_make_keepalive_http_protocol._KeepaliveProtocol.connection_made` calls `super().connection_made`, `super`, `transport.get_extra_info`, `sock.setsockopt`; returns `None`.", "documented": false, "public": false, "addressable": false, "line": 6568, "end_line": 6587, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6568-L6587", "decorators": [], "calls": [ "super().connection_made", "super", "transport.get_extra_info", "sock.setsockopt", "hasattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "main", "qualname": "main", "full_name": "vllm_mlx.server.main", "kind": "function", "signature": "def main()", "parameters": [], "return_annotation": "not annotated", "docstring": "Run the server.", "summary": "Run the server.", "implementation": "Function `main` calls `create_parser`, `parser.parse_args`, `_metrics.configure`, `RateLimiter`.", "documented": true, "public": true, "addressable": true, "line": 6597, "end_line": 6708, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6597-L6708", "decorators": [], "calls": [ "create_parser", "parser.parse_args", "_metrics.configure", "RateLimiter", "logger.info", "logger.warning", "get_parser", "parser_cls", "load_embedding_model", "load_model", "uvicorn.run", "_make_keepalive_http_protocol" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "create_parser", "qualname": "create_parser", "full_name": "vllm_mlx.server.create_parser", "kind": "function", "signature": "def create_parser() -> argparse.ArgumentParser", "parameters": [], "return_annotation": "argparse.ArgumentParser", "docstring": "Create the standalone server CLI parser.", "summary": "Create the standalone server CLI parser.", "implementation": "Function `create_parser` calls `argparse.ArgumentParser`, `parser.add_argument`, `make_positive_int_arg_parser`, `list_parsers`; returns `parser`.", "documented": true, "public": true, "addressable": true, "line": 6711, "end_line": 6912, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/server.py#L6711-L6912", "decorators": [], "calls": [ "argparse.ArgumentParser", "parser.add_argument", "make_positive_int_arg_parser", "list_parsers", "', '.join", "make_json_object_arg_parser" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parser" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.specprefill", "path": "vllm_mlx/specprefill.py", "page_path": "reference/api/vllm_mlx/specprefill.md", "docstring": "SpecPrefill: Attention-based sparse prefill for MLX.\n\nFull pipeline for reducing TTFT on long prompts:\n Step 1 (score_tokens): Use a small draft model to identify important tokens\n Step 2 (sparse_prefill): Prefill target model with only selected tokens,\n preserving original positional encoding via manual RoPE\n\nUsage:\n from specprefill import score_tokens, select_chunks, sparse_prefill, cleanup_rope\n\n # 1. Score with draft model\n importance = score_tokens(draft_model, tokens)\n\n # 2. Select important token chunks\n selected = select_chunks(importance, keep_pct=0.3)\n\n # 3. Sparse prefill on target model\n target_cache = make_prompt_cache(target_model)\n logits = sparse_prefill(target_model, tokens, selected, target_cache)\n\n # 4. Generate normally using target_cache...\n\n # 5. Cleanup\n cleanup_rope(target_model)\n\nDesign notes:\n - RoPE is relative: Q_m @ K_p^T depends only on (m - p). Selected keys stored\n contiguously in the cache buffer with correct RoPE angles produce correct\n attention during decode.\n - After sparse prefill of N tokens from a total prompt of M, cache.offset = N\n but decode RoPE needs position M. The _OffsetAdjustedRoPE adds (M - N) to\n each RoPE offset call, so decode position = N + i + (M - N) = M + i.\n - GatedDeltaNet (linear attention) layers process sparse tokens through their\n conv/SSM state normally. This is lossy but acceptable per the SpecPrefill\n paper — attention layers are the primary long-range mechanism.\n\nReference: arxiv.org/abs/2502.02789 (SpecPrefill: Speculative Prefilling)", "summary": "SpecPrefill: Attention-based sparse prefill for MLX.", "line_count": 845, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L1-L845", "members": [ "_AttentionCapture", "_qwen35_extract_queries", "_llama_extract_queries", "_nemotron_h_extract_queries", "_patch_attention_for_capture", "_unpatch_attention_capture", "_prefill_draft", "_lookahead_decode", "_avg_pool1d", "_compute_importance", "score_tokens", "select_chunks", "manual_rope", "manual_rope_with_freqs", "_PositionMappedRoPE", "_OffsetAdjustedRoPE", "_get_dims", "_get_pre_scale", "_find_attention_layers", "_get_attn_module", "_get_rope", "_set_rope", "_set_attn_module", "_build_layer_to_cache_map", "sparse_prefill", "cleanup_rope" ], "symbols": [ { "name": "_AttentionCapture", "qualname": "_AttentionCapture", "full_name": "vllm_mlx.specprefill._AttentionCapture", "kind": "class", "signature": "class _AttentionCapture", "parameters": [ { "name": "original", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "buf_idx", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "query_buffer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "query_extractor", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "_AttentionCapture", "docstring": "Wrapper that captures post-RoPE query vectors and delegates to original.\n\nInstalled on attention layers during lookahead decode to capture query\nvectors for importance scoring. Supports multiple architectures via\nquery_extractor callback.", "summary": "Wrapper that captures post-RoPE query vectors and delegates to original.", "implementation": "Class `_AttentionCapture` declares 3 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 53, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L53-L73", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_AttentionCapture.__init__", "full_name": "vllm_mlx.specprefill._AttentionCapture.__init__", "kind": "method", "signature": "def __init__(self, original, buf_idx, query_buffer, query_extractor=None)", "parameters": [ { "name": "original", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "buf_idx", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "query_buffer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "query_extractor", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_AttentionCapture.__init__` updates `self._original`, `self._buf_idx`, `self._query_buffer`, `self._query_extractor`.", "implementation": "Method `_AttentionCapture.__init__` updates `self._original`, `self._buf_idx`, `self._query_buffer`, `self._query_extractor`.", "documented": false, "public": false, "addressable": true, "line": 61, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L61-L65", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._original", "self._buf_idx", "self._query_buffer", "self._query_extractor" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "_AttentionCapture.__call__", "full_name": "vllm_mlx.specprefill._AttentionCapture.__call__", "kind": "method", "signature": "def __call__(self, x, mask=None, cache=None)", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "mask", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_AttentionCapture.__call__` calls `self._query_extractor`, `self._query_buffer[self._buf_idx].append`, `self._original`; returns `self._original(x, mask=mask, cache=cache)`.", "implementation": "Method `_AttentionCapture.__call__` calls `self._query_extractor`, `self._query_buffer[self._buf_idx].append`, `self._original`; returns `self._original(x, mask=mask, cache=cache)`.", "documented": false, "public": false, "addressable": true, "line": 67, "end_line": 70, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L67-L70", "decorators": [], "calls": [ "self._query_extractor", "self._query_buffer[self._buf_idx].append", "self._original" ], "state_reads": [ "self._query_extractor", "self._original", "self._query_buffer", "self._buf_idx" ], "state_writes": [], "raises": [], "return_expressions": [ "self._original(x, mask=mask, cache=cache)" ], "awaits": false, "yields": false }, { "name": "__getattr__", "qualname": "_AttentionCapture.__getattr__", "full_name": "vllm_mlx.specprefill._AttentionCapture.__getattr__", "kind": "method", "signature": "def __getattr__(self, name)", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_AttentionCapture.__getattr__` calls `getattr`; returns `getattr(self._original, name)`.", "implementation": "Method `_AttentionCapture.__getattr__` calls `getattr`; returns `getattr(self._original, name)`.", "documented": false, "public": false, "addressable": true, "line": 72, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L72-L73", "decorators": [], "calls": [ "getattr" ], "state_reads": [ "self._original" ], "state_writes": [], "raises": [], "return_expressions": [ "getattr(self._original, name)" ], "awaits": false, "yields": false }, { "name": "_qwen35_extract_queries", "qualname": "_qwen35_extract_queries", "full_name": "vllm_mlx.specprefill._qwen35_extract_queries", "kind": "function", "signature": "def _qwen35_extract_queries(attn, x, cache=None)", "parameters": [ { "name": "attn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Extract post-RoPE queries from Qwen3.5 attention (gate split + q_norm).\n\nQwen3.5 q_proj output is 2x wider: [queries, gate]. We split, normalize,\nthen apply RoPE.", "summary": "Extract post-RoPE queries from Qwen3.5 attention (gate split + q_norm).", "implementation": "Function `_qwen35_extract_queries` calls `attn.q_proj`, `mx.split`, `q_out.reshape`, `attn.q_norm(queries).transpose`; returns `queries`.", "documented": true, "public": false, "addressable": true, "line": 76, "end_line": 92, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L76-L92", "decorators": [], "calls": [ "attn.q_proj", "mx.split", "q_out.reshape", "attn.q_norm(queries).transpose", "attn.q_norm", "attn.rope" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "queries" ], "awaits": false, "yields": false }, { "name": "_llama_extract_queries", "qualname": "_llama_extract_queries", "full_name": "vllm_mlx.specprefill._llama_extract_queries", "kind": "function", "signature": "def _llama_extract_queries(attn, x, cache=None)", "parameters": [ { "name": "attn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Extract post-RoPE queries from standard transformer attention.\n\nStandard architecture: q_proj → reshape → RoPE. No gate, no q_norm.\nWorks for Llama 3.x, Mistral, Gemma, GPT-OSS, and other GQA models.", "summary": "Extract post-RoPE queries from standard transformer attention.", "implementation": "Function `_llama_extract_queries` calls `getattr`, `attn.q_proj`, `queries.reshape(B, L, n_heads, -1).transpose`, `queries.reshape`; returns `queries`.", "documented": true, "public": false, "addressable": true, "line": 95, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L95-L113", "decorators": [], "calls": [ "getattr", "attn.q_proj", "queries.reshape(B, L, n_heads, -1).transpose", "queries.reshape", "attn.rope" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "queries" ], "awaits": false, "yields": false }, { "name": "_nemotron_h_extract_queries", "qualname": "_nemotron_h_extract_queries", "full_name": "vllm_mlx.specprefill._nemotron_h_extract_queries", "kind": "function", "signature": "def _nemotron_h_extract_queries(attn, x, cache=None)", "parameters": [ { "name": "attn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Extract queries from Nemotron-H attention (no RoPE, no gate, no q_norm).\n\nNemotron-H attention layers have NO positional encoding — RoPE is absent.\nPositional modeling comes from Mamba2 layers. Attention is content-based only.", "summary": "Extract queries from Nemotron-H attention (no RoPE, no gate, no q_norm).", "implementation": "Function `_nemotron_h_extract_queries` calls `attn.q_proj(x).reshape(B, L, attn.num_heads, -1).transpose`, `attn.q_proj(x).reshape`, `attn.q_proj`; returns `queries`.", "documented": true, "public": false, "addressable": true, "line": 116, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L116-L125", "decorators": [], "calls": [ "attn.q_proj(x).reshape(B, L, attn.num_heads, -1).transpose", "attn.q_proj(x).reshape", "attn.q_proj" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "queries" ], "awaits": false, "yields": false }, { "name": "_patch_attention_for_capture", "qualname": "_patch_attention_for_capture", "full_name": "vllm_mlx.specprefill._patch_attention_for_capture", "kind": "function", "signature": "def _patch_attention_for_capture(model, query_buffer, query_extractor=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "query_buffer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "query_extractor", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Replace attention modules on full-attention layers with capture wrappers.\n\nSupports both `self_attn` (Qwen3.5/Llama/GPT-OSS) and `mixer`\n(Nemotron-H block_type=\"*\") attribute conventions.\n\nReturns (originals, attn_layer_indices) for cleanup.", "summary": "Replace attention modules on full-attention layers with capture wrappers.", "implementation": "Function `_patch_attention_for_capture` calls `_find_attention_layers`, `len`, `attn_indices.append`, `_get_attn_module`; returns `(originals, attn_indices)`.", "documented": true, "public": false, "addressable": true, "line": 128, "end_line": 149, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L128-L149", "decorators": [], "calls": [ "_find_attention_layers", "len", "attn_indices.append", "_get_attn_module", "_set_attn_module", "_AttentionCapture", "originals.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(originals, attn_indices)" ], "awaits": false, "yields": false }, { "name": "_unpatch_attention_capture", "qualname": "_unpatch_attention_capture", "full_name": "vllm_mlx.specprefill._unpatch_attention_capture", "kind": "function", "signature": "def _unpatch_attention_capture(model, originals)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "originals", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Restore original attention modules after capture.", "summary": "Restore original attention modules after capture.", "implementation": "Function `_unpatch_attention_capture` calls `_set_attn_module`.", "documented": true, "public": false, "addressable": true, "line": 152, "end_line": 155, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L152-L155", "decorators": [], "calls": [ "_set_attn_module" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_prefill_draft", "qualname": "_prefill_draft", "full_name": "vllm_mlx.specprefill._prefill_draft", "kind": "function", "signature": "def _prefill_draft(model, tokens, cache, step_size=2048, cancel_check=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "step_size", "kind": "positional or keyword", "annotation": "not annotated", "default": "2048", "required": false, "description": "Optional positional or keyword input; defaults to `2048`." }, { "name": "cancel_check", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Prefill prompt tokens into cache. Returns logits from last token.", "summary": "Prefill prompt tokens into cache.", "implementation": "Function `_prefill_draft` calls `isinstance`, `mx.array`, `len`, `cancel_check`; returns `logits`.", "documented": true, "public": false, "addressable": true, "line": 158, "end_line": 175, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L158-L175", "decorators": [], "calls": [ "isinstance", "mx.array", "len", "cancel_check", "min", "model", "mx.eval", "mx.clear_cache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "logits" ], "awaits": false, "yields": false }, { "name": "_lookahead_decode", "qualname": "_lookahead_decode", "full_name": "vllm_mlx.specprefill._lookahead_decode", "kind": "function", "signature": "def _lookahead_decode(model, first_logits, cache, n_steps, temp=0.6, top_p=0.95, cancel_check=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "first_logits", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_steps", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "temp", "kind": "positional or keyword", "annotation": "not annotated", "default": "0.6", "required": false, "description": "Optional positional or keyword input; defaults to `0.6`." }, { "name": "top_p", "kind": "positional or keyword", "annotation": "not annotated", "default": "0.95", "required": false, "description": "Optional positional or keyword input; defaults to `0.95`." }, { "name": "cancel_check", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Run n_steps autoregressive decode, returning generated token ids.\n\nQuery vectors are captured by the monkey-patched attention layers.", "summary": "Run n_steps autoregressive decode, returning generated token ids.", "implementation": "Function `_lookahead_decode` calls `make_sampler`, `cancel_check`, `sampler`, `mx.eval`; returns `generated`.", "documented": true, "public": false, "addressable": true, "line": 178, "end_line": 204, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L178-L204", "decorators": [], "calls": [ "make_sampler", "cancel_check", "sampler", "mx.eval", "y.item", "range", "model", "y.reshape", "generated.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "generated" ], "awaits": false, "yields": false }, { "name": "_avg_pool1d", "qualname": "_avg_pool1d", "full_name": "vllm_mlx.specprefill._avg_pool1d", "kind": "function", "signature": "def _avg_pool1d(x, kernel_size)", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "(..., M) input" }, { "name": "kernel_size", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "window size (odd for centered)" } ], "return_annotation": "not annotated", "docstring": "1D average pooling along last axis via prefix-sum.\n\nArgs:\n x: (..., M) input\n kernel_size: window size (odd for centered)\n\nReturns:\n (..., M) pooled (same size, zero-padded at edges)", "summary": "1D average pooling along last axis via prefix-sum.", "implementation": "Function `_avg_pool1d` calls `mx.pad`, `mx.zeros`, `mx.concatenate`, `mx.cumsum`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 207, "end_line": 223, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L207-L223", "decorators": [], "calls": [ "mx.pad", "mx.zeros", "mx.concatenate", "mx.cumsum" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "x", "(prefix[..., kernel_size:] - prefix[..., :-kernel_size]) / kernel_size" ], "awaits": false, "yields": false }, { "name": "_compute_importance", "qualname": "_compute_importance", "full_name": "vllm_mlx.specprefill._compute_importance", "kind": "function", "signature": "def _compute_importance(query_buffer, attn_caches, n_prompt, n_attn_heads, n_kv_heads, pool_kernel=13)", "parameters": [ { "name": "query_buffer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attn_caches", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_prompt", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_attn_heads", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "n_kv_heads", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "pool_kernel", "kind": "positional or keyword", "annotation": "not annotated", "default": "13", "required": false, "description": "Optional positional or keyword input; defaults to `13`." } ], "return_annotation": "not annotated", "docstring": "Compute per-token importance from captured queries and cached keys.\n\nAggregation (SpecPrefill paper):\n 1. softmax(Q @ K^T / sqrt(d)) per head, per layer, per lookahead token\n 2. avg_pool1d smoothing\n 3. max across (layers × heads)\n 4. mean across lookahead tokens\n\nReturns: (n_prompt,) importance scores.", "summary": "Compute per-token importance from captured queries and cached keys.", "implementation": "Function `_compute_importance` calls `enumerate`, `mx.concatenate`, `mx.repeat`, `expanded_keys.transpose`; can raise `RuntimeError`; returns `importance`.", "documented": true, "public": false, "addressable": true, "line": 226, "end_line": 271, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L226-L271", "decorators": [], "calls": [ "enumerate", "mx.concatenate", "mx.repeat", "expanded_keys.transpose", "mx.softmax", "scores.astype", "all_scores.append", "weights.squeeze", "RuntimeError", "_avg_pool1d", "mx.max", "mx.mean" ], "state_reads": [], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "importance" ], "awaits": false, "yields": false }, { "name": "score_tokens", "qualname": "score_tokens", "full_name": "vllm_mlx.specprefill.score_tokens", "kind": "function", "signature": "def score_tokens(model, tokens, n_lookahead=8, pool_kernel=13, temp=0.6, top_p=0.95, prefill_step_size=2048, query_extractor=None, cancel_check=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Draft model (small, fast — e.g. 4B)" }, { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "list or mx.array of token IDs" }, { "name": "n_lookahead", "kind": "positional or keyword", "annotation": "not annotated", "default": "8", "required": false, "description": "decode steps for query capture (default 8)" }, { "name": "pool_kernel", "kind": "positional or keyword", "annotation": "not annotated", "default": "13", "required": false, "description": "smoothing kernel for avg_pool1d (default 13, 0=disable)" }, { "name": "temp", "kind": "positional or keyword", "annotation": "not annotated", "default": "0.6", "required": false, "description": "sampling temperature for lookahead (default 0.6)" }, { "name": "top_p", "kind": "positional or keyword", "annotation": "not annotated", "default": "0.95", "required": false, "description": "top-p for lookahead (default 0.95)" }, { "name": "prefill_step_size", "kind": "positional or keyword", "annotation": "not annotated", "default": "2048", "required": false, "description": "chunk size for draft prefill (default 2048)" }, { "name": "query_extractor", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "function(attn, x, cache) → queries tensor." }, { "name": "cancel_check", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Score token importance using attention-based analysis on a draft model.\n\nRuns the full scoring pipeline:\n 1. Prefill the draft model with all tokens\n 2. N lookahead decode steps, capturing query vectors from attention layers\n 3. Compute importance: Q_lookahead @ K_prompt^T, aggregated across heads/layers\n\nThe draft model's cache is created internally and discarded after scoring.\n\nArgs:\n model: Draft model (small, fast — e.g. 4B)\n tokens: list or mx.array of token IDs\n n_lookahead: decode steps for query capture (default 8)\n pool_kernel: smoothing kernel for avg_pool1d (default 13, 0=disable)\n temp: sampling temperature for lookahead (default 0.6)\n top_p: top-p for lookahead (default 0.95)\n prefill_step_size: chunk size for draft prefill (default 2048)\n query_extractor: function(attn, x, cache) → queries tensor.\n Default: _qwen35_extract_queries. Use _llama_extract_queries for\n standard Llama/Mistral/Gemma models.\n\nReturns:\n importance: (M,) mx.array of per-token importance scores", "summary": "Score token importance using attention-based analysis on a draft model.", "implementation": "Function `score_tokens` calls `isinstance`, `tokens.tolist`, `len`, `_find_attention_layers`; returns `importance`.", "documented": true, "public": true, "addressable": true, "line": 274, "end_line": 396, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L274-L396", "decorators": [], "calls": [ "isinstance", "tokens.tolist", "len", "_find_attention_layers", "_get_attn_module", "getattr", "_EXTRACTOR_REGISTRY.get", "_get_rope", "make_prompt_cache", "_prefill_draft", "range", "_patch_attention_for_capture", "_lookahead_decode", "mx.eval", "_unpatch_attention_capture", "_build_layer_to_cache_map", "cancel_check", "_compute_importance", "mx.clear_cache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "importance" ], "awaits": false, "yields": false }, { "name": "select_chunks", "qualname": "select_chunks", "full_name": "vllm_mlx.specprefill.select_chunks", "kind": "function", "signature": "def select_chunks(importance, keep_pct=0.3, chunk_size=32, backbone_pct=0.0)", "parameters": [ { "name": "importance", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "(M,) per-token importance scores" }, { "name": "keep_pct", "kind": "positional or keyword", "annotation": "not annotated", "default": "0.3", "required": false, "description": "fraction of chunks to keep (default 0.3)" }, { "name": "chunk_size", "kind": "positional or keyword", "annotation": "not annotated", "default": "32", "required": false, "description": "tokens per chunk (default 32)" }, { "name": "backbone_pct", "kind": "positional or keyword", "annotation": "not annotated", "default": "0.0", "required": false, "description": "fraction of chunks reserved for evenly-spaced coverage" } ], "return_annotation": "not annotated", "docstring": "Select top-k% token chunks by average importance.\n\nArgs:\n importance: (M,) per-token importance scores\n keep_pct: fraction of chunks to keep (default 0.3)\n chunk_size: tokens per chunk (default 32)\n backbone_pct: fraction of chunks reserved for evenly-spaced coverage\n\nReturns:\n sorted mx.array of kept token indices", "summary": "Select top-k% token chunks by average importance.", "implementation": "Function `select_chunks` calls `mx.arange`, `math.ceil`, `max`, `range`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 399, "end_line": 467, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L399-L467", "decorators": [], "calls": [ "mx.arange", "math.ceil", "max", "range", "min", "chunk_scores.append", "mx.mean(importance[start:end]).item", "mx.mean", "set", "sorted", "selected_chunks.update", "selected_chunks.add", "round", "len", "_selected_token_count", "indices.extend", "mx.array" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.arange(M)", "mx.array(indices)" ], "awaits": false, "yields": false }, { "name": "_selected_token_count", "qualname": "select_chunks._selected_token_count", "full_name": "vllm_mlx.specprefill.select_chunks._selected_token_count", "kind": "nested function", "signature": "def _selected_token_count(chunks)", "parameters": [ { "name": "chunks", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `select_chunks._selected_token_count` calls `min`; returns `total`.", "implementation": "Nested Function `select_chunks._selected_token_count` calls `min`; returns `total`.", "documented": false, "public": false, "addressable": false, "line": 437, "end_line": 443, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L437-L443", "decorators": [], "calls": [ "min" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "total" ], "awaits": false, "yields": false }, { "name": "manual_rope", "qualname": "manual_rope", "full_name": "vllm_mlx.specprefill.manual_rope", "kind": "function", "signature": "def manual_rope(x, positions, dims, base=10000.0, scale=1.0)", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "(B, n_heads, L, head_dim) input tensor" }, { "name": "positions", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "(L,) position indices (can be non-contiguous)" }, { "name": "dims", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "number of dimensions to rotate (head_dim * partial_rotary_factor)" }, { "name": "base", "kind": "positional or keyword", "annotation": "not annotated", "default": "10000.0", "required": false, "description": "RoPE base frequency (default 10000.0)" }, { "name": "scale", "kind": "positional or keyword", "annotation": "not annotated", "default": "1.0", "required": false, "description": "position scale divisor (default 1.0, higher = compressed positions)" } ], "return_annotation": "not annotated", "docstring": "Apply RoPE at arbitrary (non-contiguous) positions.\n\nUses non-traditional (interleaved) layout matching Qwen3.5:\nrotates first `dims` dimensions as pairs [0,half), [half,dims),\npasses through [dims:] unchanged.\n\nArgs:\n x: (B, n_heads, L, head_dim) input tensor\n positions: (L,) position indices (can be non-contiguous)\n dims: number of dimensions to rotate (head_dim * partial_rotary_factor)\n base: RoPE base frequency (default 10000.0)\n scale: position scale divisor (default 1.0, higher = compressed positions)\n\nReturns:\n (B, n_heads, L, head_dim) with RoPE applied", "summary": "Apply RoPE at arbitrary (non-contiguous) positions.", "implementation": "Function `manual_rope` calls `mx.arange`, `positions.astype`, `mx.cos`, `mx.sin`; returns `mx.concatenate([rotated, x_pass], axis=-1)`.", "documented": true, "public": true, "addressable": true, "line": 480, "end_line": 508, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L480-L508", "decorators": [], "calls": [ "mx.arange", "positions.astype", "mx.cos", "mx.sin", "mx.concatenate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.concatenate([rotated, x_pass], axis=-1)" ], "awaits": false, "yields": false }, { "name": "manual_rope_with_freqs", "qualname": "manual_rope_with_freqs", "full_name": "vllm_mlx.specprefill.manual_rope_with_freqs", "kind": "function", "signature": "def manual_rope_with_freqs(x, positions, dims, freqs, pre_scale=1.0)", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "positions", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "dims", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "freqs", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "pre_scale", "kind": "positional or keyword", "annotation": "not annotated", "default": "1.0", "required": false, "description": "Optional positional or keyword input; defaults to `1.0`." } ], "return_annotation": "not annotated", "docstring": "Apply RoPE at arbitrary positions using pre-computed frequencies.\n\nFor custom RoPE variants (Llama3, Yarn, SuScaled) that store _freqs.", "summary": "Apply RoPE at arbitrary positions using pre-computed frequencies.", "implementation": "Function `manual_rope_with_freqs` calls `(1.0 / freqs).astype`, `positions[:, None].astype`, `mx.cos`, `mx.sin`; returns `mx.concatenate([rotated, x_pass], axis=-1)`.", "documented": true, "public": true, "addressable": true, "line": 511, "end_line": 528, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L511-L528", "decorators": [], "calls": [ "(1.0 / freqs).astype", "positions[:, None].astype", "mx.cos", "mx.sin", "mx.concatenate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "mx.concatenate([rotated, x_pass], axis=-1)" ], "awaits": false, "yields": false }, { "name": "_PositionMappedRoPE", "qualname": "_PositionMappedRoPE", "full_name": "vllm_mlx.specprefill._PositionMappedRoPE", "kind": "class", "signature": "class _PositionMappedRoPE", "parameters": [ { "name": "original_rope", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "all_positions", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache_start", "kind": "positional or keyword", "annotation": "not annotated", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "_PositionMappedRoPE", "docstring": "Wraps a RoPE module to apply rotation at non-contiguous positions.\n\nUsed during sparse prefill. The `offset` parameter from the cache tells us\nwhich slice of the position array to use for the current chunk:\n positions = all_positions[(offset - cache_start) : (offset - cache_start) + L]\n\nWhen composing with a pre-populated cache (e.g., system KV cache), cache_start\nis the initial cache offset so indexing into the position array is correct.", "summary": "Wraps a RoPE module to apply rotation at non-contiguous positions.", "implementation": "Class `_PositionMappedRoPE` declares 2 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 536, "end_line": 571, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L536-L571", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_PositionMappedRoPE.__init__", "full_name": "vllm_mlx.specprefill._PositionMappedRoPE.__init__", "kind": "method", "signature": "def __init__(self, original_rope, all_positions, cache_start=0)", "parameters": [ { "name": "original_rope", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "all_positions", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache_start", "kind": "positional or keyword", "annotation": "not annotated", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_PositionMappedRoPE.__init__` updates `self._original`, `self._all_positions`, `self._cache_start`, `self._has_custom_freqs`; calls `hasattr`, `_get_dims`, `_get_pre_scale`.", "implementation": "Method `_PositionMappedRoPE.__init__` updates `self._original`, `self._all_positions`, `self._cache_start`, `self._has_custom_freqs`; calls `hasattr`, `_get_dims`, `_get_pre_scale`.", "documented": false, "public": false, "addressable": true, "line": 547, "end_line": 561, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L547-L561", "decorators": [], "calls": [ "hasattr", "_get_dims", "_get_pre_scale" ], "state_reads": [ "self._has_custom_freqs" ], "state_writes": [ "self._original", "self._all_positions", "self._cache_start", "self._has_custom_freqs", "self._freqs", "self._dims", "self._pre_scale", "self._base", "self._scale" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "_PositionMappedRoPE.__call__", "full_name": "vllm_mlx.specprefill._PositionMappedRoPE.__call__", "kind": "method", "signature": "def __call__(self, x, offset=0)", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "offset", "kind": "positional or keyword", "annotation": "not annotated", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_PositionMappedRoPE.__call__` calls `manual_rope_with_freqs`, `manual_rope`; has 2 explicit return paths.", "implementation": "Method `_PositionMappedRoPE.__call__` calls `manual_rope_with_freqs`, `manual_rope`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 563, "end_line": 571, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L563-L571", "decorators": [], "calls": [ "manual_rope_with_freqs", "manual_rope" ], "state_reads": [ "self._cache_start", "self._all_positions", "self._has_custom_freqs", "self._dims", "self._freqs", "self._pre_scale", "self._base", "self._scale" ], "state_writes": [], "raises": [], "return_expressions": [ "manual_rope_with_freqs(x, positions, self._dims, self._freqs, pre_scale=self._pre_scale)", "manual_rope(x, positions, self._dims, base=self._base, scale=self._scale)" ], "awaits": false, "yields": false }, { "name": "_OffsetAdjustedRoPE", "qualname": "_OffsetAdjustedRoPE", "full_name": "vllm_mlx.specprefill._OffsetAdjustedRoPE", "kind": "class", "signature": "class _OffsetAdjustedRoPE", "parameters": [ { "name": "original_rope", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "adjustment", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_OffsetAdjustedRoPE", "docstring": "Wraps a RoPE module to add a constant offset for decode after sparse prefill.\n\nAfter sparse prefill of N tokens from a prompt of M total tokens:\n cache.offset = N + i (i = decode step)\n desired RoPE position = M + i\n adjustment = M - N\n\nSo: RoPE(x, offset = cache.offset + adjustment) = RoPE(x, M + i)", "summary": "Wraps a RoPE module to add a constant offset for decode after sparse prefill.", "implementation": "Class `_OffsetAdjustedRoPE` declares 2 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 574, "end_line": 590, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L574-L590", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_OffsetAdjustedRoPE.__init__", "full_name": "vllm_mlx.specprefill._OffsetAdjustedRoPE.__init__", "kind": "method", "signature": "def __init__(self, original_rope, adjustment)", "parameters": [ { "name": "original_rope", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "adjustment", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_OffsetAdjustedRoPE.__init__` updates `self._original`, `self._adjustment`.", "implementation": "Method `_OffsetAdjustedRoPE.__init__` updates `self._original`, `self._adjustment`.", "documented": false, "public": false, "addressable": true, "line": 585, "end_line": 587, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L585-L587", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._original", "self._adjustment" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__call__", "qualname": "_OffsetAdjustedRoPE.__call__", "full_name": "vllm_mlx.specprefill._OffsetAdjustedRoPE.__call__", "kind": "method", "signature": "def __call__(self, x, offset=0)", "parameters": [ { "name": "x", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "offset", "kind": "positional or keyword", "annotation": "not annotated", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_OffsetAdjustedRoPE.__call__` calls `self._original`; returns `self._original(x, offset=offset + self._adjustment)`.", "implementation": "Method `_OffsetAdjustedRoPE.__call__` calls `self._original`; returns `self._original(x, offset=offset + self._adjustment)`.", "documented": false, "public": false, "addressable": true, "line": 589, "end_line": 590, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L589-L590", "decorators": [], "calls": [ "self._original" ], "state_reads": [ "self._original", "self._adjustment" ], "state_writes": [], "raises": [], "return_expressions": [ "self._original(x, offset=offset + self._adjustment)" ], "awaits": false, "yields": false }, { "name": "_get_dims", "qualname": "_get_dims", "full_name": "vllm_mlx.specprefill._get_dims", "kind": "function", "signature": "def _get_dims(rope_module)", "parameters": [ { "name": "rope_module", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Extract rotary dimensions from any RoPE variant.", "summary": "Extract rotary dimensions from any RoPE variant.", "implementation": "Function `_get_dims` calls `hasattr`, `getattr`, `ValueError`, `type`; can raise `ValueError`; returns `getattr(rope_module, attr)`.", "documented": true, "public": false, "addressable": true, "line": 598, "end_line": 603, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L598-L603", "decorators": [], "calls": [ "hasattr", "getattr", "ValueError", "type" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "getattr(rope_module, attr)" ], "awaits": false, "yields": false }, { "name": "_get_pre_scale", "qualname": "_get_pre_scale", "full_name": "vllm_mlx.specprefill._get_pre_scale", "kind": "function", "signature": "def _get_pre_scale(rope_module)", "parameters": [ { "name": "rope_module", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Extract pre-scale factor from custom RoPE variants (SuScaled, Yarn).", "summary": "Extract pre-scale factor from custom RoPE variants (SuScaled, Yarn).", "implementation": "Function `_get_pre_scale` calls `hasattr`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 606, "end_line": 612, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L606-L612", "decorators": [], "calls": [ "hasattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "rope_module.mscale", "rope_module._scale", "1.0" ], "awaits": false, "yields": false }, { "name": "_find_attention_layers", "qualname": "_find_attention_layers", "full_name": "vllm_mlx.specprefill._find_attention_layers", "kind": "function", "signature": "def _find_attention_layers(model)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Find all full-attention layers across architectures.\n\nSupports:\n - Qwen3.5 / Llama / GPT-OSS: layers with `self_attn` attribute\n - Nemotron-H: layers with `block_type == \"*\"` (attention blocks use `mixer`)\n\nReturns list of (layer_idx, layer) tuples.", "summary": "Find all full-attention layers across architectures.", "implementation": "Function `_find_attention_layers` calls `enumerate`, `hasattr`, `results.append`, `getattr`; returns `results`.", "documented": true, "public": false, "addressable": true, "line": 615, "end_line": 630, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L615-L630", "decorators": [], "calls": [ "enumerate", "hasattr", "results.append", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "_get_attn_module", "qualname": "_get_attn_module", "full_name": "vllm_mlx.specprefill._get_attn_module", "kind": "function", "signature": "def _get_attn_module(layer)", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Get the attention module from a layer (self_attn or mixer).", "summary": "Get the attention module from a layer (self_attn or mixer).", "implementation": "Function `_get_attn_module` calls `hasattr`, `getattr`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 633, "end_line": 639, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L633-L639", "decorators": [], "calls": [ "hasattr", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "layer.self_attn", "layer.mixer", "None" ], "awaits": false, "yields": false }, { "name": "_get_rope", "qualname": "_get_rope", "full_name": "vllm_mlx.specprefill._get_rope", "kind": "function", "signature": "def _get_rope(attn)", "parameters": [ { "name": "attn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Get the RoPE module from an attention layer, or None.\n\nmlx_lm models use ``self.rope``; mlx_vlm models use ``self.rotary_emb``.", "summary": "Get the RoPE module from an attention layer, or None.", "implementation": "Function `_get_rope` calls `getattr`; returns `getattr(attn, 'rope', None) or getattr(attn, 'rotary_emb', None)`.", "documented": true, "public": false, "addressable": true, "line": 642, "end_line": 647, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L642-L647", "decorators": [], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "getattr(attn, 'rope', None) or getattr(attn, 'rotary_emb', None)" ], "awaits": false, "yields": false }, { "name": "_set_rope", "qualname": "_set_rope", "full_name": "vllm_mlx.specprefill._set_rope", "kind": "function", "signature": "def _set_rope(attn, rope_module)", "parameters": [ { "name": "attn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "rope_module", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Set the RoPE module on an attention layer.", "summary": "Set the RoPE module on an attention layer.", "implementation": "Function `_set_rope` calls `hasattr`.", "documented": true, "public": false, "addressable": true, "line": 650, "end_line": 655, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L650-L655", "decorators": [], "calls": [ "hasattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_set_attn_module", "qualname": "_set_attn_module", "full_name": "vllm_mlx.specprefill._set_attn_module", "kind": "function", "signature": "def _set_attn_module(layer, module)", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "module", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Set the attention module on a layer (self_attn or mixer).", "summary": "Set the attention module on a layer (self_attn or mixer).", "implementation": "Function `_set_attn_module` calls `hasattr`, `getattr`.", "documented": true, "public": false, "addressable": true, "line": 658, "end_line": 663, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L658-L663", "decorators": [], "calls": [ "hasattr", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_build_layer_to_cache_map", "qualname": "_build_layer_to_cache_map", "full_name": "vllm_mlx.specprefill._build_layer_to_cache_map", "kind": "function", "signature": "def _build_layer_to_cache_map(model)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Build mapping from model layer index to cache index.\n\nStandard models (Qwen3.5, Llama, GPT-OSS): one cache entry per layer,\nso the mapping is identity (layer_idx → layer_idx).\n\nNemotron-H: only M (Mamba2) and * (attention) layers have cache entries.\nMLP (-) and MoE (E) layers get no cache. The mapping is compacted.\n\nReturns dict {layer_idx: cache_idx}.", "summary": "Build mapping from model layer index to cache index.", "implementation": "Function `_build_layer_to_cache_map` calls `any`, `hasattr`, `range`, `len`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 666, "end_line": 690, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L666-L690", "decorators": [], "calls": [ "any", "hasattr", "range", "len", "enumerate", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{i: i for i in range(len(model.layers))}", "layer_to_cache" ], "awaits": false, "yields": false }, { "name": "sparse_prefill", "qualname": "sparse_prefill", "full_name": "vllm_mlx.specprefill.sparse_prefill", "kind": "function", "signature": "def sparse_prefill(model, tokens, selected_indices, cache, step_size=2048, position_offset=0, cancel_check=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Language model with .layers property (TextModel or VLM Model)" }, { "name": "tokens", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "(M,) all prompt token IDs (mx.array or list)" }, { "name": "selected_indices", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "(N,) sorted indices into tokens to keep (mx.array or list)" }, { "name": "cache", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "list of KVCache/ArraysCache from make_prompt_cache()" }, { "name": "step_size", "kind": "positional or keyword", "annotation": "not annotated", "default": "2048", "required": false, "description": "chunk size for processing (default 2048)" }, { "name": "position_offset", "kind": "positional or keyword", "annotation": "not annotated", "default": "0", "required": false, "description": "added to selected_indices for RoPE positions (default 0). Use when the cache already has tokens from a prior prefill (e.g., system prompt KV cache with S tokens → position_offset=S)." }, { "name": "cancel_check", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Prefill the model cache with selected tokens at their original positions.\n\nRuns the model forward on only the selected tokens while preserving their\noriginal positional encoding via manual RoPE. After this call, the cache\ncontains KV entries with correct RoPE positions, and attention layers have\n_OffsetAdjustedRoPE installed for correct decode positioning.\n\nArgs:\n model: Language model with .layers property (TextModel or VLM Model)\n tokens: (M,) all prompt token IDs (mx.array or list)\n selected_indices: (N,) sorted indices into tokens to keep (mx.array or list)\n cache: list of KVCache/ArraysCache from make_prompt_cache()\n step_size: chunk size for processing (default 2048)\n position_offset: added to selected_indices for RoPE positions (default 0).\n Use when the cache already has tokens from a prior prefill (e.g.,\n system prompt KV cache with S tokens → position_offset=S).\n\nReturns:\n logits: (1, 1, vocab_size) from the last selected token\n\nSide effects:\n - Populates cache with KV for selected tokens\n - Installs _OffsetAdjustedRoPE on attention layers for decode\n - Call cleanup_rope(model) after generation to restore original RoPE", "summary": "Prefill the model cache with selected tokens at their original positions.", "implementation": "Function `sparse_prefill` calls `isinstance`, `mx.array`, `type`, `max`; returns `logits`.", "documented": true, "public": true, "addressable": true, "line": 698, "end_line": 827, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L698-L827", "decorators": [], "calls": [ "isinstance", "mx.array", "type", "max", "getattr", "set", "range", "selected_indices.tolist", "sorted", "selected_indices.astype", "_find_attention_layers", "_build_layer_to_cache_map", "hasattr", "_get_attn_module", "_get_rope", "_set_rope", "_PositionMappedRoPE", "int", "cancel_check", "min", "model", "mx.eval", "mx.clear_cache", "_OffsetAdjustedRoPE" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "logits" ], "awaits": false, "yields": false }, { "name": "cleanup_rope", "qualname": "cleanup_rope", "full_name": "vllm_mlx.specprefill.cleanup_rope", "kind": "function", "signature": "def cleanup_rope(model)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Restore original RoPE on all attention layers.\n\nCall this after generation is complete to remove _OffsetAdjustedRoPE\nwrappers installed by sparse_prefill(). No-op for architectures\nwithout RoPE (e.g. Nemotron-H).", "summary": "Restore original RoPE on all attention layers.", "implementation": "Function `cleanup_rope` calls `_find_attention_layers`, `_get_attn_module`, `_get_rope`, `isinstance`.", "documented": true, "public": true, "addressable": true, "line": 830, "end_line": 845, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/specprefill.py#L830-L845", "decorators": [], "calls": [ "_find_attention_layers", "_get_attn_module", "_get_rope", "isinstance", "_set_rope" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.ssd_cache", "path": "vllm_mlx/ssd_cache.py", "page_path": "reference/api/vllm_mlx/ssd_cache.md", "docstring": "SSD KV cache tiering for vllm-mlx.\n\nThis module provides a cold-tier disk cache that sits behind\nMemoryAwarePrefixCache. Evicted entries spill to NVMe instead of being\ndiscarded, and cold-tier fetches reload from disk asynchronously with\nRAM budget reservation before the read completes.\n\nKey design:\n- SQLite for atomic metadata index (no mutable JSON)\n- Async writer thread for non-blocking spills\n- Per-layer serializer interface for hybrid cache types\n- Atomic temp-file + rename writes for crash consistency\n- Metrics exposed from day one", "summary": "SSD KV cache tiering for vllm-mlx.", "line_count": 1248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L1-L1248", "members": [ "logger", "_BYTES_PER_MB", "_BYTES_PER_GB", "_PREFIX_FILTER_TOKENS", "SSDCacheConfig", "SSDCacheStats", "_tokens_to_blob", "_blob_to_tokens", "_tokens_hash", "_prefix_hash", "SSDIndex", "SERIALIZER_SUPPORT_MATRIX", "LayerSerializer", "_mx_to_numpy_safe", "KVCacheSerializer", "ArraysCacheSerializer", "get_serializer_for_layer", "SSDCacheTier" ], "symbols": [ { "name": "SSDCacheConfig", "qualname": "SSDCacheConfig", "full_name": "vllm_mlx.ssd_cache.SSDCacheConfig", "kind": "class", "signature": "class SSDCacheConfig", "parameters": [ { "name": "cache_dir", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "max_size_gb", "kind": "field", "annotation": "float", "default": "10.0", "required": false, "description": "Optional constructor field; defaults to `10.0`." }, { "name": "max_entries", "kind": "field", "annotation": "int", "default": "10000", "required": false, "description": "Optional constructor field; defaults to `10000`." }, { "name": "file_permissions", "kind": "field", "annotation": "int", "default": "384", "required": false, "description": "Optional constructor field; defaults to `384`." }, { "name": "dir_permissions", "kind": "field", "annotation": "int", "default": "448", "required": false, "description": "Optional constructor field; defaults to `448`." }, { "name": "spill_queue_size", "kind": "field", "annotation": "int", "default": "64", "required": false, "description": "Optional constructor field; defaults to `64`." }, { "name": "retention_seconds", "kind": "field", "annotation": "int | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "SSDCacheConfig", "docstring": "Configuration for SSD cache tier.\n\nAttributes:\n cache_dir: Directory for SSD cache files. None = auto-detect\n (~/.cache/vllm-mlx/ssd_cache/{model}/).\n max_size_gb: Maximum total size of SSD cache in GB.\n max_entries: Maximum number of entries in SSD cache.\n file_permissions: Unix permission bits for cache data files.\n dir_permissions: Unix permission bits for cache directories.\n spill_queue_size: Max pending spill operations before dropping.\n retention_seconds: Optional max age for cache entries (None = no expiry).", "summary": "Configuration for SSD cache tier.", "implementation": "Class `SSDCacheConfig` declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 43, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L43-L78", "decorators": [ "dataclass(frozen=True)" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__post_init__", "qualname": "SSDCacheConfig.__post_init__", "full_name": "vllm_mlx.ssd_cache.SSDCacheConfig.__post_init__", "kind": "method", "signature": "def __post_init__(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `SSDCacheConfig.__post_init__` calls `ValueError`; can raise `ValueError`.", "implementation": "Method `SSDCacheConfig.__post_init__` calls `ValueError`; can raise `ValueError`.", "documented": false, "public": false, "addressable": true, "line": 65, "end_line": 73, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L65-L73", "decorators": [], "calls": [ "ValueError" ], "state_reads": [ "self.max_size_gb", "self.max_entries", "self.spill_queue_size" ], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "max_size_bytes", "qualname": "SSDCacheConfig.max_size_bytes", "full_name": "vllm_mlx.ssd_cache.SSDCacheConfig.max_size_bytes", "kind": "method", "signature": "def max_size_bytes(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Maximum cache size in bytes.", "summary": "Maximum cache size in bytes.", "implementation": "Method `SSDCacheConfig.max_size_bytes` calls `int`; returns `int(self.max_size_gb * _BYTES_PER_GB)`.", "documented": true, "public": true, "addressable": true, "line": 76, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L76-L78", "decorators": [ "property" ], "calls": [ "int" ], "state_reads": [ "self.max_size_gb" ], "state_writes": [], "raises": [], "return_expressions": [ "int(self.max_size_gb * _BYTES_PER_GB)" ], "awaits": false, "yields": false }, { "name": "SSDCacheStats", "qualname": "SSDCacheStats", "full_name": "vllm_mlx.ssd_cache.SSDCacheStats", "kind": "class", "signature": "class SSDCacheStats", "parameters": [ { "name": "spill_count", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "spill_bytes", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "ssd_hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "ssd_misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "reload_latency_sum", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "reload_bytes", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "promotion_failures", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "SSDCacheStats", "docstring": "Statistics for SSD cache tier — exposed from day one.\n\nAttributes:\n spill_count: Number of entries spilled to SSD.\n spill_bytes: Total bytes written to SSD.\n ssd_hits: Number of successful SSD cache lookups.\n ssd_misses: Number of SSD cache lookup misses.\n reload_latency_sum: Sum of reload latencies in seconds.\n reload_bytes: Total bytes read from SSD.\n promotion_failures: Number of failed promotions (RAM budget exhausted).", "summary": "Statistics for SSD cache tier — exposed from day one.", "implementation": "Class `SSDCacheStats` declares 1 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 82, "end_line": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L82-L123", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "SSDCacheStats.to_dict", "full_name": "vllm_mlx.ssd_cache.SSDCacheStats.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Return spill, lookup, reload, and promotion statistics.", "summary": "Return spill, lookup, reload, and promotion statistics.", "implementation": "Method `SSDCacheStats.to_dict` calls `round`; returns `{'spill_count': self.spill_count, 'spill_bytes': self.spill_bytes, 'ssd_hits': self.ssd_hits, 'ssd_misses': self.ssd_mi…`.", "documented": true, "public": true, "addressable": true, "line": 103, "end_line": 123, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L103-L123", "decorators": [], "calls": [ "round" ], "state_reads": [ "self.ssd_hits", "self.ssd_misses", "self.reload_latency_sum", "self.spill_count", "self.spill_bytes", "self.reload_bytes", "self.promotion_failures" ], "state_writes": [], "raises": [], "return_expressions": [ "{'spill_count': self.spill_count, 'spill_bytes': self.spill_bytes, 'ssd_hits': self.ssd_hits, 'ssd_misses': self.ssd_mi…" ], "awaits": false, "yields": false }, { "name": "_tokens_to_blob", "qualname": "_tokens_to_blob", "full_name": "vllm_mlx.ssd_cache._tokens_to_blob", "kind": "function", "signature": "def _tokens_to_blob(tokens: tuple[int, ...]) -> bytes", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bytes", "docstring": "Serialize token tuple to a compact binary blob for SQLite storage.\n\nUses the full token sequence as a binary blob for prefix matching.", "summary": "Serialize token tuple to a compact binary blob for SQLite storage.", "implementation": "Function `_tokens_to_blob` calls `_array.array`, `arr.tobytes`; returns `arr.tobytes()`.", "documented": true, "public": false, "addressable": true, "line": 126, "end_line": 132, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L126-L132", "decorators": [], "calls": [ "_array.array", "arr.tobytes" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "arr.tobytes()" ], "awaits": false, "yields": false }, { "name": "_blob_to_tokens", "qualname": "_blob_to_tokens", "full_name": "vllm_mlx.ssd_cache._blob_to_tokens", "kind": "function", "signature": "def _blob_to_tokens(blob: bytes) -> tuple[int, ...]", "parameters": [ { "name": "blob", "kind": "positional or keyword", "annotation": "bytes", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[int, ...]", "docstring": "Deserialize binary blob back to token tuple.", "summary": "Deserialize binary blob back to token tuple.", "implementation": "Function `_blob_to_tokens` calls `_array.array`, `arr.frombytes`, `tuple`; returns `tuple(arr)`.", "documented": true, "public": false, "addressable": true, "line": 135, "end_line": 139, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L135-L139", "decorators": [], "calls": [ "_array.array", "arr.frombytes", "tuple" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "tuple(arr)" ], "awaits": false, "yields": false }, { "name": "_tokens_hash", "qualname": "_tokens_hash", "full_name": "vllm_mlx.ssd_cache._tokens_hash", "kind": "function", "signature": "def _tokens_hash(tokens: tuple[int, ...]) -> str", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Compute SHA-256 hex digest of a token sequence for use as primary key.", "summary": "Compute SHA-256 hex digest of a token sequence for use as primary key.", "implementation": "Function `_tokens_hash` calls `hashlib.sha256(_tokens_to_blob(tokens)).hexdigest`, `hashlib.sha256`, `_tokens_to_blob`; returns `hashlib.sha256(_tokens_to_blob(tokens)).hexdigest()`.", "documented": true, "public": false, "addressable": true, "line": 142, "end_line": 144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L142-L144", "decorators": [], "calls": [ "hashlib.sha256(_tokens_to_blob(tokens)).hexdigest", "hashlib.sha256", "_tokens_to_blob" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hashlib.sha256(_tokens_to_blob(tokens)).hexdigest()" ], "awaits": false, "yields": false }, { "name": "_prefix_hash", "qualname": "_prefix_hash", "full_name": "vllm_mlx.ssd_cache._prefix_hash", "kind": "function", "signature": "def _prefix_hash(tokens: tuple[int, ...]) -> str", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Hash the bounded token prefix used to prefilter prefix lookups.", "summary": "Hash the bounded token prefix used to prefilter prefix lookups.", "implementation": "Function `_prefix_hash` calls `_tokens_hash`; returns `_tokens_hash(tokens[:_PREFIX_FILTER_TOKENS])`.", "documented": true, "public": false, "addressable": true, "line": 147, "end_line": 149, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L147-L149", "decorators": [], "calls": [ "_tokens_hash" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_tokens_hash(tokens[:_PREFIX_FILTER_TOKENS])" ], "awaits": false, "yields": false }, { "name": "SSDIndex", "qualname": "SSDIndex", "full_name": "vllm_mlx.ssd_cache.SSDIndex", "kind": "class", "signature": "class SSDIndex", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "SSDIndex", "docstring": "SQLite-backed index for SSD cache entries.\n\nUses SQLite for atomic metadata operations instead of mutable JSON.\nThe token sequence is stored as a binary blob for prefix-searchable\nrepresentation. The primary key is a SHA-256 hash of the token sequence.\n\nThread safety: All operations are serialized through a threading.Lock.\nThe SQLite connection uses WAL mode for concurrent read/write safety.", "summary": "SQLite-backed index for SSD cache entries.", "implementation": "Class `SSDIndex` declares 14 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 152, "end_line": 405, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L152-L405", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "SSDIndex.__init__", "full_name": "vllm_mlx.ssd_cache.SSDIndex.__init__", "kind": "method", "signature": "def __init__(self, cache_dir: str) -> None", "parameters": [ { "name": "cache_dir", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `SSDIndex.__init__` updates `self._cache_dir`, `self._db_lock`, `self._conn`, `self._conn.row_factory`; calls `threading.Lock`, `os.path.join`, `sqlite3.connect`, `self._conn.execute`.", "implementation": "Method `SSDIndex.__init__` updates `self._cache_dir`, `self._db_lock`, `self._conn`, `self._conn.row_factory`; calls `threading.Lock`, `os.path.join`, `sqlite3.connect`, `self._conn.execute`.", "documented": false, "public": false, "addressable": true, "line": 165, "end_line": 173, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L165-L173", "decorators": [], "calls": [ "threading.Lock", "os.path.join", "sqlite3.connect", "self._conn.execute", "self._create_tables" ], "state_reads": [ "self._conn.execute", "self._conn", "self._create_tables" ], "state_writes": [ "self._cache_dir", "self._db_lock", "self._conn", "self._conn.row_factory" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_create_tables", "qualname": "SSDIndex._create_tables", "full_name": "vllm_mlx.ssd_cache.SSDIndex._create_tables", "kind": "method", "signature": "def _create_tables(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `SSDIndex._create_tables` calls `self._conn.executescript`, `self._ensure_column`, `self._conn.execute`, `cur.fetchone`.", "implementation": "Method `SSDIndex._create_tables` calls `self._conn.executescript`, `self._ensure_column`, `self._conn.execute`, `cur.fetchone`.", "documented": false, "public": false, "addressable": true, "line": 175, "end_line": 213, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L175-L213", "decorators": [], "calls": [ "self._conn.executescript", "self._ensure_column", "self._conn.execute", "cur.fetchone", "self._backfill_prefix_hashes", "self._conn.commit" ], "state_reads": [ "self._conn.executescript", "self._conn", "self._ensure_column", "self._conn.execute", "self._SCHEMA_VERSION", "self._backfill_prefix_hashes", "self._conn.commit" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_ensure_column", "qualname": "SSDIndex._ensure_column", "full_name": "vllm_mlx.ssd_cache.SSDIndex._ensure_column", "kind": "method", "signature": "def _ensure_column(self, table: str, column: str, definition: str) -> None", "parameters": [ { "name": "table", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "column", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "definition", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `SSDIndex._ensure_column` calls `self._conn.execute`, `cur.fetchall`.", "implementation": "Method `SSDIndex._ensure_column` calls `self._conn.execute`, `cur.fetchall`.", "documented": false, "public": false, "addressable": true, "line": 215, "end_line": 218, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L215-L218", "decorators": [], "calls": [ "self._conn.execute", "cur.fetchall" ], "state_reads": [ "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_backfill_prefix_hashes", "qualname": "SSDIndex._backfill_prefix_hashes", "full_name": "vllm_mlx.ssd_cache.SSDIndex._backfill_prefix_hashes", "kind": "method", "signature": "def _backfill_prefix_hashes(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `SSDIndex._backfill_prefix_hashes` calls `self._conn.execute`, `cur.fetchall`, `_blob_to_tokens`, `_prefix_hash`.", "implementation": "Method `SSDIndex._backfill_prefix_hashes` calls `self._conn.execute`, `cur.fetchall`, `_blob_to_tokens`, `_prefix_hash`.", "documented": false, "public": false, "addressable": true, "line": 220, "end_line": 230, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L220-L230", "decorators": [], "calls": [ "self._conn.execute", "cur.fetchall", "_blob_to_tokens", "_prefix_hash" ], "state_reads": [ "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "insert_entry", "qualname": "SSDIndex.insert_entry", "full_name": "vllm_mlx.ssd_cache.SSDIndex.insert_entry", "kind": "method", "signature": "def insert_entry(self, tokens_key: tuple[int, ...], file_path: str, memory_bytes: int, num_tokens: int) -> None", "parameters": [ { "name": "tokens_key", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "memory_bytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_tokens", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Insert or replace a cache entry in the index.", "summary": "Insert or replace a cache entry in the index.", "implementation": "Method `SSDIndex.insert_entry` calls `time.time`, `_tokens_hash`, `_prefix_hash`, `_tokens_to_blob`.", "documented": true, "public": true, "addressable": true, "line": 232, "end_line": 263, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L232-L263", "decorators": [], "calls": [ "time.time", "_tokens_hash", "_prefix_hash", "_tokens_to_blob", "self._conn.execute", "self._conn.commit" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn", "self._conn.commit" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "lookup_exact", "qualname": "SSDIndex.lookup_exact", "full_name": "vllm_mlx.ssd_cache.SSDIndex.lookup_exact", "kind": "method", "signature": "def lookup_exact(self, tokens_key: tuple[int, ...]) -> dict | None", "parameters": [ { "name": "tokens_key", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict | None", "docstring": "Look up an exact token sequence. Returns dict or None.", "summary": "Look up an exact token sequence.", "implementation": "Method `SSDIndex.lookup_exact` calls `_tokens_hash`, `self._conn.execute`, `cur.fetchone`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 265, "end_line": 280, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L265-L280", "decorators": [], "calls": [ "_tokens_hash", "self._conn.execute", "cur.fetchone" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "{'file_path': row['file_path'], 'memory_bytes': row['memory_bytes'], 'num_tokens': row['num_tokens']}" ], "awaits": false, "yields": false }, { "name": "lookup_prefix", "qualname": "SSDIndex.lookup_prefix", "full_name": "vllm_mlx.ssd_cache.SSDIndex.lookup_prefix", "kind": "method", "signature": "def lookup_prefix(self, query_tokens: tuple[int, ...]) -> list[dict]", "parameters": [ { "name": "query_tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Find entries whose token sequence is a prefix of query_tokens.\n\nUses a bounded token-prefix hash to avoid scanning all entries, then\ncompares the full stored token blob against the corresponding prefix of\nquery_tokens.\n\nReturns list of dicts sorted by num_tokens descending (longest prefix first).", "summary": "Find entries whose token sequence is a prefix of query_tokens.", "implementation": "Method `SSDIndex.lookup_prefix` calls `len`, `_tokens_to_blob`, `_tokens_hash`, `range`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 282, "end_line": 324, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L282-L324", "decorators": [], "calls": [ "len", "_tokens_to_blob", "_tokens_hash", "range", "min", "','.join", "self._conn.execute", "cur.fetchall", "results.append" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [ "[]", "results" ], "awaits": false, "yields": false }, { "name": "delete_entry", "qualname": "SSDIndex.delete_entry", "full_name": "vllm_mlx.ssd_cache.SSDIndex.delete_entry", "kind": "method", "signature": "def delete_entry(self, tokens_key: tuple[int, ...]) -> None", "parameters": [ { "name": "tokens_key", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Delete an entry by token sequence.", "summary": "Delete an entry by token sequence.", "implementation": "Method `SSDIndex.delete_entry` calls `_tokens_hash`, `self._conn.execute`, `self._conn.commit`.", "documented": true, "public": true, "addressable": true, "line": 326, "end_line": 333, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L326-L333", "decorators": [], "calls": [ "_tokens_hash", "self._conn.execute", "self._conn.commit" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn", "self._conn.commit" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_lru", "qualname": "SSDIndex.get_lru", "full_name": "vllm_mlx.ssd_cache.SSDIndex.get_lru", "kind": "method", "signature": "def get_lru(self, limit: int=10) -> list[dict]", "parameters": [ { "name": "limit", "kind": "positional or keyword", "annotation": "int", "default": "10", "required": false, "description": "Optional positional or keyword input; defaults to `10`." } ], "return_annotation": "list[dict]", "docstring": "Get the least recently used entries, ordered oldest first.", "summary": "Get the least recently used entries, ordered oldest first.", "implementation": "Method `SSDIndex.get_lru` calls `self._conn.execute`, `cur.fetchall`, `results.append`; returns `results`.", "documented": true, "public": true, "addressable": true, "line": 335, "end_line": 355, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L335-L355", "decorators": [], "calls": [ "self._conn.execute", "cur.fetchall", "results.append" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "get_total_bytes", "qualname": "SSDIndex.get_total_bytes", "full_name": "vllm_mlx.ssd_cache.SSDIndex.get_total_bytes", "kind": "method", "signature": "def get_total_bytes(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get total memory_bytes across all entries.", "summary": "Get total memory_bytes across all entries.", "implementation": "Method `SSDIndex.get_total_bytes` calls `self._conn.execute`, `cur.fetchone`; returns `cur.fetchone()[0]`.", "documented": true, "public": true, "addressable": true, "line": 357, "end_line": 363, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L357-L363", "decorators": [], "calls": [ "self._conn.execute", "cur.fetchone" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [ "cur.fetchone()[0]" ], "awaits": false, "yields": false }, { "name": "get_entry_count", "qualname": "SSDIndex.get_entry_count", "full_name": "vllm_mlx.ssd_cache.SSDIndex.get_entry_count", "kind": "method", "signature": "def get_entry_count(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get number of entries in the index.", "summary": "Get number of entries in the index.", "implementation": "Method `SSDIndex.get_entry_count` calls `self._conn.execute`, `cur.fetchone`; returns `cur.fetchone()[0]`.", "documented": true, "public": true, "addressable": true, "line": 365, "end_line": 369, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L365-L369", "decorators": [], "calls": [ "self._conn.execute", "cur.fetchone" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [ "cur.fetchone()[0]" ], "awaits": false, "yields": false }, { "name": "touch", "qualname": "SSDIndex.touch", "full_name": "vllm_mlx.ssd_cache.SSDIndex.touch", "kind": "method", "signature": "def touch(self, tokens_key: tuple[int, ...]) -> None", "parameters": [ { "name": "tokens_key", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Update accessed_at timestamp for an entry (marks as recently used).", "summary": "Update accessed_at timestamp for an entry (marks as recently used).", "implementation": "Method `SSDIndex.touch` calls `_tokens_hash`, `self._conn.execute`, `time.time`, `self._conn.commit`.", "documented": true, "public": true, "addressable": true, "line": 371, "end_line": 379, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L371-L379", "decorators": [], "calls": [ "_tokens_hash", "self._conn.execute", "time.time", "self._conn.commit" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn", "self._conn.commit" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "all_entries", "qualname": "SSDIndex.all_entries", "full_name": "vllm_mlx.ssd_cache.SSDIndex.all_entries", "kind": "method", "signature": "def all_entries(self) -> list[dict]", "parameters": [], "return_annotation": "list[dict]", "docstring": "Return all entries (for startup reconciliation).", "summary": "Return all entries (for startup reconciliation).", "implementation": "Method `SSDIndex.all_entries` calls `self._conn.execute`, `cur.fetchall`, `results.append`; returns `results`.", "documented": true, "public": true, "addressable": true, "line": 381, "end_line": 400, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L381-L400", "decorators": [], "calls": [ "self._conn.execute", "cur.fetchall", "results.append" ], "state_reads": [ "self._db_lock", "self._conn.execute", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "close", "qualname": "SSDIndex.close", "full_name": "vllm_mlx.ssd_cache.SSDIndex.close", "kind": "method", "signature": "def close(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Close the SQLite connection.", "summary": "Close the SQLite connection.", "implementation": "Method `SSDIndex.close` calls `self._conn.close`.", "documented": true, "public": true, "addressable": true, "line": 402, "end_line": 405, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L402-L405", "decorators": [], "calls": [ "self._conn.close" ], "state_reads": [ "self._db_lock", "self._conn.close", "self._conn" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "LayerSerializer", "qualname": "LayerSerializer", "full_name": "vllm_mlx.ssd_cache.LayerSerializer", "kind": "class", "signature": "class LayerSerializer(ABC)", "parameters": [], "return_annotation": "LayerSerializer", "docstring": "Interface for per-layer cache serialization.\n\nSpill is split across two threads: ``snapshot_layer`` runs on the\nproducer (request handler) thread so the mx→numpy materialization\nhappens where the per-request Stream(gpu, N) is registered;\n``serialize_layer`` then runs on the SSD writer thread with numpy only.", "summary": "Interface for per-layer cache serialization.", "implementation": "Class `LayerSerializer` derives from `ABC` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 419, "end_line": 446, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L419-L446", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "snapshot_layer", "qualname": "LayerSerializer.snapshot_layer", "full_name": "vllm_mlx.ssd_cache.LayerSerializer.snapshot_layer", "kind": "method", "signature": "def snapshot_layer(self, layer: Any) -> dict[str, Any]", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Producer-thread CPU snapshot of an MLX-backed cache layer.", "summary": "Producer-thread CPU snapshot of an MLX-backed cache layer.", "implementation": "Method `LayerSerializer.snapshot_layer` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 429, "end_line": 431, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L429-L431", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "serialize_layer", "qualname": "LayerSerializer.serialize_layer", "full_name": "vllm_mlx.ssd_cache.LayerSerializer.serialize_layer", "kind": "method", "signature": "def serialize_layer(self, snapshot: dict[str, Any], layer_idx: int, file_path: str) -> dict[str, Any]", "parameters": [ { "name": "snapshot", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "layer_idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Writer-thread: persist a snapshot to safetensors at file_path.\n\nReturns metadata dict with at least 'layer_type'.", "summary": "Writer-thread: persist a snapshot to safetensors at file_path.", "implementation": "Method `LayerSerializer.serialize_layer` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 434, "end_line": 441, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L434-L441", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "deserialize_layer", "qualname": "LayerSerializer.deserialize_layer", "full_name": "vllm_mlx.ssd_cache.LayerSerializer.deserialize_layer", "kind": "method", "signature": "def deserialize_layer(self, file_path: str, metadata: dict[str, Any]) -> dict", "parameters": [ { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "metadata", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Read a layer back from disk. Returns layer-state dict.", "summary": "Read a layer back from disk.", "implementation": "Method `LayerSerializer.deserialize_layer` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 444, "end_line": 446, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L444-L446", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_mx_to_numpy_safe", "qualname": "_mx_to_numpy_safe", "full_name": "vllm_mlx.ssd_cache._mx_to_numpy_safe", "kind": "function", "signature": "def _mx_to_numpy_safe(arr: Any) -> tuple[np.ndarray, str | None]", "parameters": [ { "name": "arr", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[np.ndarray, str | None]", "docstring": "mx.array → np.ndarray, upcasting numpy-unsupported dtypes (bf16) to fp32.\n\nReturns (numpy_array, original_dtype_name_or_None). The name is only set\nwhen an upcast happened, so the SSD-promote path can cast back.", "summary": "mx.array → np.ndarray, upcasting numpy-unsupported dtypes (bf16) to fp32.", "implementation": "Function `_mx_to_numpy_safe` calls `np.array`, `str`, `str(arr.dtype).rsplit`, `arr.astype`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 449, "end_line": 467, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L449-L467", "decorators": [], "calls": [ "np.array", "str", "str(arr.dtype).rsplit", "arr.astype", "mx.eval" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(np.array(arr), None)", "(np.array(upcast), original_dtype)" ], "awaits": false, "yields": false }, { "name": "KVCacheSerializer", "qualname": "KVCacheSerializer", "full_name": "vllm_mlx.ssd_cache.KVCacheSerializer", "kind": "class", "signature": "class KVCacheSerializer(LayerSerializer)", "parameters": [], "return_annotation": "KVCacheSerializer", "docstring": "Serializer for KVCache and RotatingKVCache layers.\n\nHandles layers with .keys, .values, .offset attributes.\nRotatingKVCache also has .max_size, .keep, .step, ._idx.", "summary": "Serializer for KVCache and RotatingKVCache layers.", "implementation": "Class `KVCacheSerializer` derives from `LayerSerializer` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 470, "end_line": 564, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L470-L564", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "snapshot_layer", "qualname": "KVCacheSerializer.snapshot_layer", "full_name": "vllm_mlx.ssd_cache.KVCacheSerializer.snapshot_layer", "kind": "method", "signature": "def snapshot_layer(self, layer: Any) -> dict[str, Any]", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Copy a KV cache layer into NumPy-backed writer-thread data.", "summary": "Copy a KV cache layer into NumPy-backed writer-thread data.", "implementation": "Method `KVCacheSerializer.snapshot_layer` calls `_mx_to_numpy_safe`, `getattr`, `hasattr`; returns `snapshot`.", "documented": true, "public": true, "addressable": true, "line": 481, "end_line": 515, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L481-L515", "decorators": [], "calls": [ "_mx_to_numpy_safe", "getattr", "hasattr" ], "state_reads": [ "self._ROTATING_ATTRS" ], "state_writes": [], "raises": [], "return_expressions": [ "snapshot" ], "awaits": false, "yields": false }, { "name": "serialize_layer", "qualname": "KVCacheSerializer.serialize_layer", "full_name": "vllm_mlx.ssd_cache.KVCacheSerializer.serialize_layer", "kind": "method", "signature": "def serialize_layer(self, snapshot: dict[str, Any], layer_idx: int, file_path: str) -> dict[str, Any]", "parameters": [ { "name": "snapshot", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "layer_idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Write one KV layer to safetensors and return reconstruction metadata.", "summary": "Write one KV layer to safetensors and return reconstruction metadata.", "implementation": "Method `KVCacheSerializer.serialize_layer` calls `save_file`; returns `metadata`.", "documented": true, "public": true, "addressable": true, "line": 517, "end_line": 542, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L517-L542", "decorators": [], "calls": [ "save_file" ], "state_reads": [ "self._ROTATING_ATTRS" ], "state_writes": [], "raises": [], "return_expressions": [ "metadata" ], "awaits": false, "yields": false }, { "name": "deserialize_layer", "qualname": "KVCacheSerializer.deserialize_layer", "full_name": "vllm_mlx.ssd_cache.KVCacheSerializer.deserialize_layer", "kind": "method", "signature": "def deserialize_layer(self, file_path: str, metadata: dict[str, Any]) -> dict", "parameters": [ { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "metadata", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Load one KV layer as arrays plus cache reconstruction metadata.", "summary": "Load one KV layer as arrays plus cache reconstruction metadata.", "implementation": "Method `KVCacheSerializer.deserialize_layer` calls `load_file`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 544, "end_line": 564, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L544-L564", "decorators": [], "calls": [ "load_file" ], "state_reads": [ "self._ROTATING_ATTRS" ], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "ArraysCacheSerializer", "qualname": "ArraysCacheSerializer", "full_name": "vllm_mlx.ssd_cache.ArraysCacheSerializer", "kind": "class", "signature": "class ArraysCacheSerializer(LayerSerializer)", "parameters": [], "return_annotation": "ArraysCacheSerializer", "docstring": "Serializer for ArraysCache (Mamba/linear attention) layers.\n\nHandles layers with .state attribute containing a list of arrays.", "summary": "Serializer for ArraysCache (Mamba/linear attention) layers.", "implementation": "Class `ArraysCacheSerializer` derives from `LayerSerializer` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 567, "end_line": 627, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L567-L627", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "snapshot_layer", "qualname": "ArraysCacheSerializer.snapshot_layer", "full_name": "vllm_mlx.ssd_cache.ArraysCacheSerializer.snapshot_layer", "kind": "method", "signature": "def snapshot_layer(self, layer: Any) -> dict[str, Any]", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Copy an arrays-cache state into NumPy-backed writer-thread data.", "summary": "Copy an arrays-cache state into NumPy-backed writer-thread data.", "implementation": "Method `ArraysCacheSerializer.snapshot_layer` calls `_mx_to_numpy_safe`, `state_np.append`, `original_dtypes.append`, `any`; returns `snapshot`.", "documented": true, "public": true, "addressable": true, "line": 573, "end_line": 587, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L573-L587", "decorators": [], "calls": [ "_mx_to_numpy_safe", "state_np.append", "original_dtypes.append", "any" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "snapshot" ], "awaits": false, "yields": false }, { "name": "serialize_layer", "qualname": "ArraysCacheSerializer.serialize_layer", "full_name": "vllm_mlx.ssd_cache.ArraysCacheSerializer.serialize_layer", "kind": "method", "signature": "def serialize_layer(self, snapshot: dict[str, Any], layer_idx: int, file_path: str) -> dict[str, Any]", "parameters": [ { "name": "snapshot", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "layer_idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Write arrays-cache state to safetensors and return its metadata.", "summary": "Write arrays-cache state to safetensors and return its metadata.", "implementation": "Method `ArraysCacheSerializer.serialize_layer` calls `enumerate`, `save_file`, `len`; returns `metadata`.", "documented": true, "public": true, "addressable": true, "line": 589, "end_line": 610, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L589-L610", "decorators": [], "calls": [ "enumerate", "save_file", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "metadata" ], "awaits": false, "yields": false }, { "name": "deserialize_layer", "qualname": "ArraysCacheSerializer.deserialize_layer", "full_name": "vllm_mlx.ssd_cache.ArraysCacheSerializer.deserialize_layer", "kind": "method", "signature": "def deserialize_layer(self, file_path: str, metadata: dict[str, Any]) -> dict", "parameters": [ { "name": "file_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "metadata", "kind": "positional or keyword", "annotation": "dict[str, Any]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict", "docstring": "Load arrays-cache state and any original dtype hints.", "summary": "Load arrays-cache state and any original dtype hints.", "implementation": "Method `ArraysCacheSerializer.deserialize_layer` calls `load_file`, `range`, `state.append`; returns `result`.", "documented": true, "public": true, "addressable": true, "line": 612, "end_line": 627, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L612-L627", "decorators": [], "calls": [ "load_file", "range", "state.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result" ], "awaits": false, "yields": false }, { "name": "get_serializer_for_layer", "qualname": "get_serializer_for_layer", "full_name": "vllm_mlx.ssd_cache.get_serializer_for_layer", "kind": "function", "signature": "def get_serializer_for_layer(layer: Any) -> LayerSerializer", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "LayerSerializer", "docstring": "Return the appropriate serializer for a cache layer.\n\nDispatches based on duck-typing:\n- If layer has .keys and .values and .offset -> KVCacheSerializer\n- If layer has .state and it's a list -> ArraysCacheSerializer\n\nRaises ValueError for unsupported layer types.", "summary": "Return the appropriate serializer for a cache layer.", "implementation": "Function `get_serializer_for_layer` calls `hasattr`, `KVCacheSerializer`, `isinstance`, `getattr`; can raise `ValueError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 630, "end_line": 646, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L630-L646", "decorators": [], "calls": [ "hasattr", "KVCacheSerializer", "isinstance", "getattr", "ArraysCacheSerializer", "ValueError", "type", "list", "SERIALIZER_SUPPORT_MATRIX.keys" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "KVCacheSerializer()", "ArraysCacheSerializer()" ], "awaits": false, "yields": false }, { "name": "SSDCacheTier", "qualname": "SSDCacheTier", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier", "kind": "class", "signature": "class SSDCacheTier", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "SSDCacheConfig", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "SSDCacheTier", "docstring": "Cold-tier disk cache for KV cache entries.\n\nManages a SQLite-indexed on-disk cache directory. Evicted RAM entries\nare spilled here via an async writer thread. Cold-tier fetches reload\nfrom disk asynchronously with RAM budget reservation.\n\nDirectory layout::\n\n cache_dir/\n index.db # SQLite metadata index\n data/ # safetensors files per entry\n {hash}/ # one directory per entry\n layer_0.safetensors\n layer_1.safetensors\n manifest.json # per-entry layer metadata", "summary": "Cold-tier disk cache for KV cache entries.", "implementation": "Class `SSDCacheTier` declares 15 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 649, "end_line": 1248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L649-L1248", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "SSDCacheTier.__init__", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.__init__", "kind": "method", "signature": "def __init__(self, config: SSDCacheConfig) -> None", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "SSDCacheConfig", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "", "summary": "Method `SSDCacheTier.__init__` updates `self._config`, `self._closed`, `self._writer_thread`, `self._cache_dir`; calls `ValueError`, `os.path.join`, `os.makedirs`, `SSDIndex`; can raise `ValueError`.", "implementation": "Method `SSDCacheTier.__init__` updates `self._config`, `self._closed`, `self._writer_thread`, `self._cache_dir`; calls `ValueError`, `os.path.join`, `os.makedirs`, `SSDIndex`; can raise `ValueError`.", "documented": false, "public": false, "addressable": true, "line": 667, "end_line": 705, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L667-L705", "decorators": [], "calls": [ "ValueError", "os.path.join", "os.makedirs", "SSDIndex", "SSDCacheStats", "threading.Lock", "queue.Queue", "threading.Event", "getattr", "index.close", "logger.exception" ], "state_reads": [ "self._cache_dir", "self._data_dir" ], "state_writes": [ "self._config", "self._closed", "self._writer_thread", "self._cache_dir", "self._data_dir", "self._index", "self._stats", "self._lock", "self._spill_queue", "self._writer_stop" ], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_entry_hash", "qualname": "SSDCacheTier._entry_hash", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier._entry_hash", "kind": "method", "signature": "def _entry_hash(tokens: tuple[int, ...]) -> str", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Compute deterministic hash for a token sequence.", "summary": "Compute deterministic hash for a token sequence.", "implementation": "Method `SSDCacheTier._entry_hash` calls `_tokens_hash`; returns `_tokens_hash(tokens)`.", "documented": true, "public": false, "addressable": true, "line": 708, "end_line": 710, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L708-L710", "decorators": [ "staticmethod" ], "calls": [ "_tokens_hash" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_tokens_hash(tokens)" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "SSDCacheTier.get_stats", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Return current SSD cache statistics.", "summary": "Return current SSD cache statistics.", "implementation": "Method `SSDCacheTier.get_stats` calls `self._stats.to_dict`; returns `self._stats.to_dict()`.", "documented": true, "public": true, "addressable": true, "line": 712, "end_line": 714, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L712-L714", "decorators": [], "calls": [ "self._stats.to_dict" ], "state_reads": [ "self._stats.to_dict", "self._stats" ], "state_writes": [], "raises": [], "return_expressions": [ "self._stats.to_dict()" ], "awaits": false, "yields": false }, { "name": "start_writer", "qualname": "SSDCacheTier.start_writer", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.start_writer", "kind": "method", "signature": "def start_writer(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Start the background spill writer thread.", "summary": "Start the background spill writer thread.", "implementation": "Method `SSDCacheTier.start_writer` updates `self._writer_thread`; calls `self._writer_stop.clear`, `threading.Thread`, `self._writer_thread.start`, `logger.info`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 716, "end_line": 725, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L716-L725", "decorators": [], "calls": [ "self._writer_stop.clear", "threading.Thread", "self._writer_thread.start", "logger.info" ], "state_reads": [ "self._writer_thread", "self._writer_stop.clear", "self._writer_stop", "self._writer_loop", "self._writer_thread.start" ], "state_writes": [ "self._writer_thread" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_writer_loop", "qualname": "SSDCacheTier._writer_loop", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier._writer_loop", "kind": "method", "signature": "def _writer_loop(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Drain spill queue and persist entries. Numpy-only — no MLX here.", "summary": "Drain spill queue and persist entries.", "implementation": "Method `SSDCacheTier._writer_loop` calls `self._writer_stop.is_set`, `self._spill_queue.get`, `self._write_entry`, `logger.exception`.", "documented": true, "public": false, "addressable": true, "line": 727, "end_line": 744, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L727-L744", "decorators": [], "calls": [ "self._writer_stop.is_set", "self._spill_queue.get", "self._write_entry", "logger.exception", "len" ], "state_reads": [ "self._writer_stop.is_set", "self._writer_stop", "self._spill_queue.get", "self._spill_queue", "self._write_entry" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "enqueue_spill", "qualname": "SSDCacheTier.enqueue_spill", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.enqueue_spill", "kind": "method", "signature": "def enqueue_spill(self, tokens: tuple[int, ...], cache: list[Any], memory_bytes: int) -> bool", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "cache", "kind": "positional or keyword", "annotation": "list[Any]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "memory_bytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Enqueue a cache entry for async spill to SSD.\n\nMust be called on the producer thread (the request handler that\nowns the layer's Stream(gpu, N)) — the snapshot below materializes\nMLX → numpy here so the writer thread never has to.\n\nReturns True if enqueued, False if queue is full (entry dropped).", "summary": "Enqueue a cache entry for async spill to SSD.", "implementation": "Method `SSDCacheTier.enqueue_spill` calls `any`, `_is_quantized_layer`, `isinstance`, `converted.extend`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 746, "end_line": 867, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L746-L867", "decorators": [], "calls": [ "any", "_is_quantized_layer", "isinstance", "converted.extend", "_dequantize_cache", "hasattr", "getattr", "_KVCache.__new__", "mx.dequantize", "len", "converted.append", "str(getattr(k, 'dtype', '')).endswith", "str", "k.astype", "v.astype", "mx.eval", "logger.info", "get_serializer_for_layer", "serializer.snapshot_layer", "layer_snapshots.append", "logger.exception", "self._spill_queue.put_nowait", "logger.warning" ], "state_reads": [ "self._spill_queue.put_nowait", "self._spill_queue" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_is_quantized_layer", "qualname": "SSDCacheTier.enqueue_spill._is_quantized_layer", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.enqueue_spill._is_quantized_layer", "kind": "nested function", "signature": "def _is_quantized_layer(layer)", "parameters": [ { "name": "layer", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `SSDCacheTier.enqueue_spill._is_quantized_layer` calls `isinstance`, `getattr`; has 2 explicit return paths.", "implementation": "Nested Function `SSDCacheTier.enqueue_spill._is_quantized_layer` calls `isinstance`, `getattr`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 772, "end_line": 776, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L772-L776", "decorators": [], "calls": [ "isinstance", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "isinstance(keys, (tuple, list))" ], "awaits": false, "yields": false }, { "name": "_write_entry", "qualname": "SSDCacheTier._write_entry", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier._write_entry", "kind": "method", "signature": "def _write_entry(self, tokens_key: tuple[int, ...], layer_snapshots: list[tuple[LayerSerializer, dict[str, Any]]], memory_bytes: int) -> None", "parameters": [ { "name": "tokens_key", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "layer_snapshots", "kind": "positional or keyword", "annotation": "list[tuple[LayerSerializer, dict[str, Any]]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "memory_bytes", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Atomically persist one entry (writer thread; numpy-only input).", "summary": "Atomically persist one entry (writer thread; numpy-only input).", "implementation": "Method `SSDCacheTier._write_entry` updates `self._stats.spill_count`, `self._stats.spill_bytes`; calls `self._entry_hash`, `os.path.join`, `os.path.exists`, `shutil.rmtree`.", "documented": true, "public": false, "addressable": true, "line": 869, "end_line": 944, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L869-L944", "decorators": [], "calls": [ "self._entry_hash", "os.path.join", "os.path.exists", "shutil.rmtree", "os.makedirs", "enumerate", "serializer.serialize_layer", "layer_manifests.append", "os.chmod", "os.path.getsize", "len", "open", "json.dump", "_array.array", "arr.tofile", "os.rename", "self._index.insert_entry", "logger.debug", "self._enforce_capacity" ], "state_reads": [ "self._entry_hash", "self._data_dir", "self._config.dir_permissions", "self._config", "self._config.file_permissions", "self._index.insert_entry", "self._index", "self._lock", "self._stats", "self._enforce_capacity" ], "state_writes": [ "self._stats.spill_count", "self._stats.spill_bytes" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "lookup_ssd", "qualname": "SSDCacheTier.lookup_ssd", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.lookup_ssd", "kind": "method", "signature": "def lookup_ssd(self, tokens: tuple[int, ...]) -> dict | None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict | None", "docstring": "Synchronous check whether tokens exist in SSD tier.\n\nThis is fast (SQLite lookup only, no disk I/O for data).\nCalled from synchronous fetch() to report an SSD candidate.\n\nReturns:\n Dict with entry metadata if found, None otherwise.", "summary": "Synchronous check whether tokens exist in SSD tier.", "implementation": "Method `SSDCacheTier.lookup_ssd` calls `self._index.lookup_exact`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 946, "end_line": 958, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L946-L958", "decorators": [], "calls": [ "self._index.lookup_exact" ], "state_reads": [ "self._index.lookup_exact", "self._index" ], "state_writes": [], "raises": [], "return_expressions": [ "result", "None" ], "awaits": false, "yields": false }, { "name": "lookup_ssd_prefix", "qualname": "SSDCacheTier.lookup_ssd_prefix", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.lookup_ssd_prefix", "kind": "method", "signature": "def lookup_ssd_prefix(self, tokens: tuple[int, ...]) -> dict | None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict | None", "docstring": "Find the longest prefix match in the SSD tier.\n\nReturns the longest-prefix entry metadata or None.", "summary": "Find the longest prefix match in the SSD tier.", "implementation": "Method `SSDCacheTier.lookup_ssd_prefix` calls `self._index.lookup_prefix`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 960, "end_line": 968, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L960-L968", "decorators": [], "calls": [ "self._index.lookup_prefix" ], "state_reads": [ "self._index.lookup_prefix", "self._index" ], "state_writes": [], "raises": [], "return_expressions": [ "results[0]", "None" ], "awaits": false, "yields": false }, { "name": "async_promote", "qualname": "SSDCacheTier.async_promote", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.async_promote", "kind": "method", "signature": "async def async_promote(self, tokens: tuple[int, ...], reserve_budget_fn, release_budget_fn) -> list | None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Token sequence to promote." }, { "name": "reserve_budget_fn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Callable(nbytes) -> bool. Must return True if budget is available and reserved, False otherwise." }, { "name": "release_budget_fn", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Callable(nbytes) -> None. Called to release budget on failure." } ], "return_annotation": "list | None", "docstring": "Promote an entry from SSD to RAM asynchronously.\n\nCRITICAL: Reserves RAM budget BEFORE the disk read, to avoid\nthrash when multiple promotions race.\n\nArgs:\n tokens: Token sequence to promote.\n reserve_budget_fn: Callable(nbytes) -> bool. Must return True\n if budget is available and reserved, False otherwise.\n release_budget_fn: Callable(nbytes) -> None. Called to release\n budget on failure.\n\nReturns:\n List of deserialized cache layers, or None if promotion failed.", "summary": "Promote an entry from SSD to RAM asynchronously.", "implementation": "Method `SSDCacheTier.async_promote` updates `self._stats.ssd_misses`, `self._stats.promotion_failures`, `self._stats.ssd_hits`, `self._stats.reload_latency_sum`; calls `self._index.lookup_exact`, `reserve_budget_fn`, `logger.warning`, `time.time`; awaits asynchronous work; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 970, "end_line": 1076, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L970-L1076", "decorators": [], "calls": [ "self._index.lookup_exact", "reserve_budget_fn", "logger.warning", "time.time", "asyncio.ensure_future", "asyncio.to_thread", "asyncio.shield", "release_budget_fn", "logger.exception", "sum", "os.path.getsize", "os.path.join", "range", "len", "os.path.exists", "self._index.touch", "logger.info" ], "state_reads": [ "self._index.lookup_exact", "self._index", "self._lock", "self._stats", "self._read_entry", "self._data_dir", "self._index.touch" ], "state_writes": [ "self._stats.ssd_misses", "self._stats.promotion_failures", "self._stats.ssd_hits", "self._stats.reload_latency_sum", "self._stats.reload_bytes" ], "raises": [], "return_expressions": [ "None", "cache_layers" ], "awaits": true, "yields": false }, { "name": "_read_entry", "qualname": "SSDCacheTier._read_entry", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier._read_entry", "kind": "method", "signature": "def _read_entry(self, tokens: tuple[int, ...], relative_path: str) -> list | None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "relative_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list | None", "docstring": "Read a cache entry from disk. Called from thread pool.\n\nReturns list of deserialized layer dicts, or None on corruption.", "summary": "Read a cache entry from disk.", "implementation": "Method `SSDCacheTier._read_entry` calls `os.path.join`, `open`, `json.load`, `logger.warning`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1078, "end_line": 1121, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L1078-L1121", "decorators": [], "calls": [ "os.path.join", "open", "json.load", "logger.warning", "self._quarantine_entry", "KVCacheSerializer", "ArraysCacheSerializer", "serializer.deserialize_layer", "cache_layers.append" ], "state_reads": [ "self._data_dir", "self._quarantine_entry" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "cache_layers" ], "awaits": false, "yields": false }, { "name": "_quarantine_entry", "qualname": "SSDCacheTier._quarantine_entry", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier._quarantine_entry", "kind": "method", "signature": "def _quarantine_entry(self, tokens: tuple[int, ...], relative_path: str) -> None", "parameters": [ { "name": "tokens", "kind": "positional or keyword", "annotation": "tuple[int, ...]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "relative_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Move a corrupt entry to quarantine and remove from index.", "summary": "Move a corrupt entry to quarantine and remove from index.", "implementation": "Method `SSDCacheTier._quarantine_entry` calls `os.path.join`, `os.path.exists`, `os.makedirs`, `os.path.dirname`.", "documented": true, "public": false, "addressable": true, "line": 1123, "end_line": 1142, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L1123-L1142", "decorators": [], "calls": [ "os.path.join", "os.path.exists", "os.makedirs", "os.path.dirname", "os.rename", "logger.warning", "self._index.delete_entry" ], "state_reads": [ "self._data_dir", "self._cache_dir", "self._config.dir_permissions", "self._config", "self._index.delete_entry", "self._index" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_enforce_capacity", "qualname": "SSDCacheTier._enforce_capacity", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier._enforce_capacity", "kind": "method", "signature": "def _enforce_capacity(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Evict oldest SSD entries until within capacity limits.\n\nCalled after each spill write. Removes entries by LRU order\nuntil both entry count and total bytes are within bounds.", "summary": "Evict oldest SSD entries until within capacity limits.", "implementation": "Method `SSDCacheTier._enforce_capacity` calls `self._index.get_entry_count`, `self._index.get_total_bytes`, `self._index.get_lru`, `_blob_to_tokens`.", "documented": true, "public": false, "addressable": true, "line": 1144, "end_line": 1181, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L1144-L1181", "decorators": [], "calls": [ "self._index.get_entry_count", "self._index.get_total_bytes", "self._index.get_lru", "_blob_to_tokens", "os.path.join", "os.path.exists", "shutil.rmtree", "self._index.delete_entry", "logger.debug" ], "state_reads": [ "self._index.get_entry_count", "self._index", "self._index.get_total_bytes", "self._config.max_entries", "self._config", "self._config.max_size_bytes", "self._index.get_lru", "self._data_dir", "self._index.delete_entry" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reconcile", "qualname": "SSDCacheTier.reconcile", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.reconcile", "kind": "method", "signature": "def reconcile(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Reconcile index with files on disk.\n\nRemoves index entries whose data files are missing.\nRemoves data directories not in the index.\n\nReturns number of entries cleaned up.", "summary": "Reconcile index with files on disk.", "implementation": "Method `SSDCacheTier.reconcile` calls `self._index.all_entries`, `os.path.join`, `os.path.isdir`, `os.path.exists`; returns `cleaned`.", "documented": true, "public": true, "addressable": true, "line": 1183, "end_line": 1229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L1183-L1229", "decorators": [], "calls": [ "self._index.all_entries", "os.path.join", "os.path.isdir", "os.path.exists", "_blob_to_tokens", "self._index.delete_entry", "logger.info", "os.listdir", "entry_name.endswith", "shutil.rmtree" ], "state_reads": [ "self._index.all_entries", "self._index", "self._data_dir", "self._index.delete_entry" ], "state_writes": [], "raises": [], "return_expressions": [ "cleaned" ], "awaits": false, "yields": false }, { "name": "close", "qualname": "SSDCacheTier.close", "full_name": "vllm_mlx.ssd_cache.SSDCacheTier.close", "kind": "method", "signature": "def close(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Close the SSD cache tier and release resources.", "summary": "Close the SSD cache tier and release resources.", "implementation": "Method `SSDCacheTier.close` updates `self._closed`, `self._writer_thread`; calls `self._writer_stop.set`, `self._spill_queue.put_nowait`, `self._writer_thread.join`, `self._index.close`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 1231, "end_line": 1248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/ssd_cache.py#L1231-L1248", "decorators": [], "calls": [ "self._writer_stop.set", "self._spill_queue.put_nowait", "self._writer_thread.join", "self._index.close", "logger.info" ], "state_reads": [ "self._closed", "self._writer_stop.set", "self._writer_stop", "self._writer_thread", "self._spill_queue.put_nowait", "self._spill_queue", "self._writer_thread.join", "self._index.close", "self._index" ], "state_writes": [ "self._closed", "self._writer_thread" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.text_model_from_vlm", "path": "vllm_mlx/text_model_from_vlm.py", "page_path": "reference/api/vllm_mlx/text_model_from_vlm.md", "docstring": "Construct an mlx_lm TextModel from mlx_vlm-loaded model weights.\n\nWhen mlx_vlm loads a model, it strips MTP weights in sanitize().\nThis module builds a parallel mlx_lm TextModel that:\n1. Shares backbone + lm_head weights with the vlm model (zero-copy)\n2. Loads MTP weights from safetensors on disk\n3. Provides full mlx_lm API: return_hidden, n_confirmed, mtp_forward, make_mtp_cache", "summary": "Construct an mlx_lm TextModel from mlx_vlm-loaded model weights.", "line_count": 272, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/text_model_from_vlm.py#L1-L272", "members": [ "logger", "_TEXT_MODEL_FAMILIES", "_DEFAULT_TEXT_MODEL", "_import_text_model_classes", "build_text_model", "_load_mtp_weights" ], "symbols": [ { "name": "_import_text_model_classes", "qualname": "_import_text_model_classes", "full_name": "vllm_mlx.text_model_from_vlm._import_text_model_classes", "kind": "function", "signature": "def _import_text_model_classes(model_type: str)", "parameters": [ { "name": "model_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Return ``(Model, ModelArgs)`` for a text config's ``model_type``.", "summary": "Return ``(Model, ModelArgs)`` for a text config's ``model_type``.", "implementation": "Function `_import_text_model_classes` calls `sorted`, `model_type.startswith`, `logger.debug`, `importlib.import_module`; returns `(getattr(module, model_attr), getattr(module, args_attr))`.", "documented": true, "public": false, "addressable": true, "line": 41, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/text_model_from_vlm.py#L41-L66", "decorators": [], "calls": [ "sorted", "model_type.startswith", "logger.debug", "importlib.import_module", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(getattr(module, model_attr), getattr(module, args_attr))" ], "awaits": false, "yields": false }, { "name": "build_text_model", "qualname": "build_text_model", "full_name": "vllm_mlx.text_model_from_vlm.build_text_model", "kind": "function", "signature": "def build_text_model(vlm_model: Any, model_path: str | Path) -> Any | None", "parameters": [ { "name": "vlm_model", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The mlx_vlm-loaded model (has .language_model attribute)" }, { "name": "model_path", "kind": "positional or keyword", "annotation": "str | Path", "default": "", "required": true, "description": "Path to the model directory (contains config.json + safetensors)" } ], "return_annotation": "Any | None", "docstring": "Build an mlx_lm TextModel from a vlm-loaded model's weights.\n\nArgs:\n vlm_model: The mlx_vlm-loaded model (has .language_model attribute)\n model_path: Path to the model directory (contains config.json + safetensors)\n\nReturns:\n mlx_lm TextModel with MTP support, or None on failure.", "summary": "Build an mlx_lm TextModel from a vlm-loaded model's weights.", "implementation": "Function `build_text_model` calls `Path`, `(model_path / 'config.json').exists`, `json.loads`, `(model_path / 'config.json').read_text`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/text_model_from_vlm.py#L69-L225", "decorators": [], "calls": [ "Path", "(model_path / 'config.json').exists", "json.loads", "(model_path / 'config.json').read_text", "config.get", "text_config.get", "_import_text_model_classes", "logger.debug", "TextModelArgs.from_dict", "TextModel", "mlx.utils.tree_flatten", "vlm_lm.parameters", "_load_mtp_weights", "set", "all_weight_names.update", "quantization.items", "isinstance", "nn.quantize", "quantization.get", "text_model.load_weights", "logger.info", "len", "logger.warning", "hasattr", "inject_mtp_support", "mx.eval", "text_model.mtp.parameters", "text_model.train", "text_model.modules", "module.values", "logger.error" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "text_model" ], "awaits": false, "yields": false }, { "name": "_class_predicate", "qualname": "build_text_model._class_predicate", "full_name": "vllm_mlx.text_model_from_vlm.build_text_model._class_predicate", "kind": "nested function", "signature": "def _class_predicate(path, module)", "parameters": [ { "name": "path", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "module", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `build_text_model._class_predicate` calls `hasattr`, `per_layer_overrides.items`, `key.endswith`; has 4 explicit return paths.", "implementation": "Nested Function `build_text_model._class_predicate` calls `hasattr`, `per_layer_overrides.items`, `key.endswith`; has 4 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 127, "end_line": 137, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/text_model_from_vlm.py#L127-L137", "decorators": [], "calls": [ "hasattr", "per_layer_overrides.items", "key.endswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "quantization[path]", "override", "True" ], "awaits": false, "yields": false }, { "name": "_load_mtp_weights", "qualname": "_load_mtp_weights", "full_name": "vllm_mlx.text_model_from_vlm._load_mtp_weights", "kind": "function", "signature": "def _load_mtp_weights(model_path: Path) -> list[tuple[str, mx.array]]", "parameters": [ { "name": "model_path", "kind": "positional or keyword", "annotation": "Path", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[tuple[str, mx.array]]", "docstring": "Load MTP weights from safetensors, stripping the language_model. prefix.\n\nmlx_vlm's sanitize() strips mtp.* keys during model loading,\nbut the weights are still on disk in the safetensors files.", "summary": "Load MTP weights from safetensors, stripping the language_model.", "implementation": "Function `_load_mtp_weights` calls `index_file.exists`, `json.loads`, `index_file.read_text`, `index.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 228, "end_line": 272, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/text_model_from_vlm.py#L228-L272", "decorators": [], "calls": [ "index_file.exists", "json.loads", "index_file.read_text", "index.get", "weight_map.items", "key.startswith", "key.replace", "mtp_keys.items", "shards.setdefault(shard, []).append", "shards.setdefault", "shards.items", "shard_path.exists", "logger.warning", "mx.load", "str", "weights.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "weights" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers", "path": "vllm_mlx/tool_parsers/__init__.py", "page_path": "reference/api/vllm_mlx/tool_parsers/index.md", "docstring": "Tool call parsers for vllm-mlx.\n\nThis module provides tool call parsing functionality for various model formats.\nInspired by vLLM's tool parser architecture but simplified for MLX backend.\n\nAvailable parsers:\n- auto: Auto-detecting parser that tries all formats (default)\n- mistral: Mistral models ([TOOL_CALLS] format)\n- qwen/qwen3: Qwen models ( and [Calling tool:] formats)\n- llama/llama3/llama4: Llama models ( format)\n- gemma4/gemma_4: Google Gemma 4 models (<|tool_call>call:name{} format)\n- hermes/nous: Hermes/NousResearch models\n- deepseek/deepseek_v3/deepseek_r1: DeepSeek models (unicode tokens)\n- kimi/kimi_k2/moonshot: Kimi/Moonshot models\n- granite/granite3: IBM Granite models\n- nemotron/nemotron3: NVIDIA Nemotron models\n- xlam: Salesforce xLAM models\n- functionary/meetkai: MeetKai Functionary models\n- glm47/glm4: GLM-4.7 and GLM-4.7-Flash models\n- harmony/gpt-oss: GPT-OSS models (Harmony format with channels)\n- minimax: MiniMax-M2 models\n\nUsage:\n from vllm_mlx.tool_parsers import ToolParserManager\n\n # Get a parser by name\n parser_cls = ToolParserManager.get_tool_parser(\"mistral\")\n parser = parser_cls(tokenizer)\n\n # Parse tool calls\n result = parser.extract_tool_calls(model_output)\n if result.tools_called:\n for tc in result.tool_calls:\n print(f\"Tool: {tc['name']}, Args: {tc['arguments']}\")\n\n # List available parsers\n print(ToolParserManager.list_registered())", "summary": "Tool call parsers for vllm-mlx.", "line_count": 116, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/__init__.py#L1-L116", "members": [ "get_parser_stop_tokens", "__all__" ], "symbols": [ { "name": "get_parser_stop_tokens", "qualname": "get_parser_stop_tokens", "full_name": "vllm_mlx.tool_parsers.get_parser_stop_tokens", "kind": "function", "signature": "def get_parser_stop_tokens(parser_name: str | None, user_stops: list[str] | None) -> list[str]", "parameters": [ { "name": "parser_name", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "user_stops", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Merge user-supplied stops with parser-declared extras (deduped).\n\nSome models declare end-of-generation tokens beyond the tokenizer's default\neos set — e.g. Gemma 4's ``<|tool_response>`` which signals the runtime's\nturn after a tool call. Parsers expose those via ``extra_stop_tokens``.", "summary": "Merge user-supplied stops with parser-declared extras (deduped).", "implementation": "Function `get_parser_stop_tokens` calls `list`, `ToolParserManager.get_tool_parser`, `getattr`, `stops.append`; returns `stops`.", "documented": true, "public": true, "addressable": true, "line": 68, "end_line": 88, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/__init__.py#L68-L88", "decorators": [], "calls": [ "list", "ToolParserManager.get_tool_parser", "getattr", "stops.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "stops" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.abstract_tool_parser", "path": "vllm_mlx/tool_parsers/abstract_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/abstract_tool_parser.md", "docstring": "Abstract tool parser base class and manager for vllm-mlx.\n\nInspired by vLLM's tool parser architecture but simplified for MLX backend.", "summary": "Abstract tool parser base class and manager for vllm-mlx.", "line_count": 286, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L1-L286", "members": [ "THINK_TAG_PATTERN", "IMPLICIT_THINK_PATTERN", "ExtractedToolCallInformation", "ToolParser", "ToolParserManager" ], "symbols": [ { "name": "ExtractedToolCallInformation", "qualname": "ExtractedToolCallInformation", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ExtractedToolCallInformation", "kind": "class", "signature": "class ExtractedToolCallInformation", "parameters": [ { "name": "tools_called", "kind": "field", "annotation": "bool", "default": "", "required": true, "description": "Required constructor field." }, { "name": "tool_calls", "kind": "field", "annotation": "list[dict[str, Any]]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "content", "kind": "field", "annotation": "str | None", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Information extracted from model output about tool calls.", "summary": "Information extracted from model output about tool calls.", "implementation": "Class `ExtractedToolCallInformation` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 27, "end_line": 37, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L27-L37", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ToolParser", "qualname": "ToolParser", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser", "kind": "class", "signature": "class ToolParser(ABC)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "PreTrainedTokenizerBase | None", "default": "None", "required": false, "description": "The tokenizer for the model (optional, some parsers need it)" } ], "return_annotation": "ToolParser", "docstring": "Abstract base class for tool call parsers.\n\nEach parser implementation handles a specific model's tool calling format.", "summary": "Abstract base class for tool call parsers.", "implementation": "Class `ToolParser` derives from `ABC` and declares 7 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 40, "end_line": 171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L40-L171", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "supports_native_format", "qualname": "ToolParser.supports_native_format", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser.supports_native_format", "kind": "method", "signature": "def supports_native_format(cls) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if this parser supports native tool message format.\n\nNative format means the parser's corresponding model chat template\ncan handle:\n- role=\"tool\" messages directly (not converted to role=\"user\")\n- tool_calls field on assistant messages (not converted to text)\n\nReturns:\n True if native format is supported", "summary": "Check if this parser supports native tool message format.", "implementation": "Method `ToolParser.supports_native_format` returns `cls.SUPPORTS_NATIVE_TOOL_FORMAT`.", "documented": true, "public": true, "addressable": true, "line": 60, "end_line": 72, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L60-L72", "decorators": [ "classmethod" ], "calls": [], "state_reads": [ "cls.SUPPORTS_NATIVE_TOOL_FORMAT" ], "state_writes": [], "raises": [], "return_expressions": [ "cls.SUPPORTS_NATIVE_TOOL_FORMAT" ], "awaits": false, "yields": false }, { "name": "strip_think_tags", "qualname": "ToolParser.strip_think_tags", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser.strip_think_tags", "kind": "method", "signature": "def strip_think_tags(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Model output that may contain think tags" } ], "return_annotation": "str", "docstring": "Strip think tags from text.\n\nHandles two scenarios:\n1. Full tags: ... in output\n2. Only closing tag: ... when was in prompt\n\nUsed as fallback when no reasoning parser is configured but the model\nproduces thinking tags. This prevents tool parsing failures with\nmodels that use thinking tags (e.g., Ring-Mini-Linear-2.0 with hermes).\n\nArgs:\n text: Model output that may contain think tags\n\nReturns:\n Text with think tags removed", "summary": "Strip think tags from text.", "implementation": "Method `ToolParser.strip_think_tags` calls `THINK_TAG_PATTERN.sub`, `IMPLICIT_THINK_PATTERN.sub`, `result.strip`; returns `result.strip()`.", "documented": true, "public": true, "addressable": true, "line": 75, "end_line": 101, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L75-L101", "decorators": [ "staticmethod" ], "calls": [ "THINK_TAG_PATTERN.sub", "IMPLICIT_THINK_PATTERN.sub", "result.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result.strip()" ], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "ToolParser.__init__", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer: PreTrainedTokenizerBase | None=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "PreTrainedTokenizerBase | None", "default": "None", "required": false, "description": "The tokenizer for the model (optional, some parsers need it)" } ], "return_annotation": "not annotated", "docstring": "Initialize the tool parser.\n\nArgs:\n tokenizer: The tokenizer for the model (optional, some parsers need it)", "summary": "Initialize the tool parser.", "implementation": "Method `ToolParser.__init__` updates `self.model_tokenizer`, `self.current_tool_id`, `self.prev_tool_call_arr`.", "documented": true, "public": false, "addressable": true, "line": 103, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L103-L113", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.model_tokenizer", "self.current_tool_id", "self.prev_tool_call_arr" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "vocab", "qualname": "ToolParser.vocab", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser.vocab", "kind": "method", "signature": "def vocab(self) -> dict[str, int]", "parameters": [], "return_annotation": "dict[str, int]", "docstring": "Get the tokenizer vocabulary.", "summary": "Get the tokenizer vocabulary.", "implementation": "Method `ToolParser.vocab` calls `self.model_tokenizer.get_vocab`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 116, "end_line": 120, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L116-L120", "decorators": [ "cached_property" ], "calls": [ "self.model_tokenizer.get_vocab" ], "state_reads": [ "self.model_tokenizer", "self.model_tokenizer.get_vocab" ], "state_writes": [], "raises": [], "return_expressions": [ "{}", "self.model_tokenizer.get_vocab()" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "ToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The complete model output string" }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional request context (for tool definitions, etc.)" } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete model response.\n\nArgs:\n model_output: The complete model output string\n request: Optional request context (for tool definitions, etc.)\n\nReturns:\n ExtractedToolCallInformation with parsed tool calls", "summary": "Extract tool calls from a complete model response.", "implementation": "Method `ToolParser.extract_tool_calls` can raise `NotImplementedError`.", "documented": true, "public": true, "addressable": true, "line": 123, "end_line": 136, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L123-L136", "decorators": [ "abstractmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [ "NotImplementedError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "ToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Text before this delta" }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Complete text so far" }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "New text in this chunk" }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Token IDs before this delta" }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "All token IDs so far" }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "New token IDs in this chunk" }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional request context" } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming model output.\n\nOverride this method for streaming support. Default implementation\nreturns None (no streaming support).\n\nArgs:\n previous_text: Text before this delta\n current_text: Complete text so far\n delta_text: New text in this chunk\n previous_token_ids: Token IDs before this delta\n current_token_ids: All token IDs so far\n delta_token_ids: New token IDs in this chunk\n request: Optional request context\n\nReturns:\n Delta message dict with content and/or tool_calls, or None", "summary": "Extract tool calls from streaming model output.", "implementation": "Method `ToolParser.extract_tool_calls_streaming` returns `None`.", "documented": true, "public": true, "addressable": true, "line": 138, "end_line": 166, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L138-L166", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "reset", "qualname": "ToolParser.reset", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParser.reset", "kind": "method", "signature": "def reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset parser state for a new request.", "summary": "Reset parser state for a new request.", "implementation": "Method `ToolParser.reset` updates `self.current_tool_id`, `self.prev_tool_call_arr`.", "documented": true, "public": true, "addressable": true, "line": 168, "end_line": 171, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L168-L171", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.current_tool_id", "self.prev_tool_call_arr" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ToolParserManager", "qualname": "ToolParserManager", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParserManager", "kind": "class", "signature": "class ToolParserManager", "parameters": [], "return_annotation": "ToolParserManager", "docstring": "Central registry for ToolParser implementations.\n\nSupports both eager and lazy registration of tool parsers.", "summary": "Central registry for ToolParser implementations.", "implementation": "Class `ToolParserManager` declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 174, "end_line": 286, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L174-L286", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_tool_parser", "qualname": "ToolParserManager.get_tool_parser", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParserManager.get_tool_parser", "kind": "method", "signature": "def get_tool_parser(cls, name: str) -> type[ToolParser]", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Parser name (e.g., 'mistral', 'qwen', 'llama')" } ], "return_annotation": "type[ToolParser]", "docstring": "Retrieve a registered ToolParser class by name.\n\nArgs:\n name: Parser name (e.g., 'mistral', 'qwen', 'llama')\n\nReturns:\n The ToolParser class\n\nRaises:\n KeyError: If parser not found", "summary": "Retrieve a registered ToolParser class by name.", "implementation": "Method `ToolParserManager.get_tool_parser` calls `cls._load_lazy_parser`, `KeyError`, `cls.list_registered`; can raise `KeyError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 185, "end_line": 207, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L185-L207", "decorators": [ "classmethod" ], "calls": [ "cls._load_lazy_parser", "KeyError", "cls.list_registered" ], "state_reads": [ "cls.tool_parsers", "cls.lazy_parsers", "cls._load_lazy_parser", "cls.list_registered" ], "state_writes": [], "raises": [ "KeyError" ], "return_expressions": [ "cls.tool_parsers[name]", "cls._load_lazy_parser(name)" ], "awaits": false, "yields": false }, { "name": "_load_lazy_parser", "qualname": "ToolParserManager._load_lazy_parser", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParserManager._load_lazy_parser", "kind": "method", "signature": "def _load_lazy_parser(cls, name: str) -> type[ToolParser]", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "type[ToolParser]", "docstring": "Import and register a lazily loaded parser.", "summary": "Import and register a lazily loaded parser.", "implementation": "Method `ToolParserManager._load_lazy_parser` calls `importlib.import_module`, `getattr`, `issubclass`, `TypeError`; can raise `TypeError`, `ImportError`; returns `parser_cls`.", "documented": true, "public": false, "addressable": true, "line": 210, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L210-L225", "decorators": [ "classmethod" ], "calls": [ "importlib.import_module", "getattr", "issubclass", "TypeError", "ImportError" ], "state_reads": [ "cls.lazy_parsers", "cls.tool_parsers" ], "state_writes": [], "raises": [ "TypeError", "ImportError" ], "return_expressions": [ "parser_cls" ], "awaits": false, "yields": false }, { "name": "register_module", "qualname": "ToolParserManager.register_module", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParserManager.register_module", "kind": "method", "signature": "def register_module(cls, name: str | list[str], module: type[ToolParser] | None=None, force: bool=True) -> type[ToolParser] | None", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str | list[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "module", "kind": "positional or keyword", "annotation": "type[ToolParser] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "force", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Optional positional or keyword input; defaults to `True`." } ], "return_annotation": "type[ToolParser] | None", "docstring": "Register a ToolParser class.\n\nCan be used as a decorator or direct call.\n\nUsage:\n @ToolParserManager.register_module(\"my_parser\")\n class MyToolParser(ToolParser):\n ...\n\n # Or direct registration:\n ToolParserManager.register_module(\"my_parser\", MyToolParser)", "summary": "Register a ToolParser class.", "implementation": "Method `ToolParserManager.register_module` calls `isinstance`, `issubclass`, `TypeError`, `type`; can raise `TypeError`, `KeyError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 228, "end_line": 269, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L228-L269", "decorators": [ "classmethod" ], "calls": [ "isinstance", "issubclass", "TypeError", "type", "KeyError" ], "state_reads": [ "cls.tool_parsers" ], "state_writes": [], "raises": [ "TypeError", "KeyError" ], "return_expressions": [ "module", "decorator" ], "awaits": false, "yields": false }, { "name": "decorator", "qualname": "ToolParserManager.register_module.decorator", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParserManager.register_module.decorator", "kind": "nested function", "signature": "def decorator(parser_cls: type[ToolParser]) -> type[ToolParser]", "parameters": [ { "name": "parser_cls", "kind": "positional or keyword", "annotation": "type[ToolParser]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "type[ToolParser]", "docstring": "", "summary": "Nested Function `ToolParserManager.register_module.decorator` calls `KeyError`; can raise `KeyError`; returns `parser_cls`.", "implementation": "Nested Function `ToolParserManager.register_module.decorator` calls `KeyError`; can raise `KeyError`; returns `parser_cls`.", "documented": false, "public": false, "addressable": false, "line": 262, "end_line": 267, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L262-L267", "decorators": [], "calls": [ "KeyError" ], "state_reads": [ "cls.tool_parsers" ], "state_writes": [], "raises": [ "KeyError" ], "return_expressions": [ "parser_cls" ], "awaits": false, "yields": false }, { "name": "register_lazy_module", "qualname": "ToolParserManager.register_lazy_module", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParserManager.register_lazy_module", "kind": "method", "signature": "def register_lazy_module(cls, name: str, module_path: str, class_name: str) -> None", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Parser name to register" }, { "name": "module_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Full module path (e.g., 'vllm_mlx.tool_parsers.mistral')" }, { "name": "class_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Class name within the module" } ], "return_annotation": "None", "docstring": "Register a lazy module mapping for deferred loading.\n\nArgs:\n name: Parser name to register\n module_path: Full module path (e.g., 'vllm_mlx.tool_parsers.mistral')\n class_name: Class name within the module", "summary": "Register a lazy module mapping for deferred loading.", "implementation": "Method `ToolParserManager.register_lazy_module` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 272, "end_line": 281, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L272-L281", "decorators": [ "classmethod" ], "calls": [], "state_reads": [ "cls.lazy_parsers" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "list_registered", "qualname": "ToolParserManager.list_registered", "full_name": "vllm_mlx.tool_parsers.abstract_tool_parser.ToolParserManager.list_registered", "kind": "method", "signature": "def list_registered(cls) -> list[str]", "parameters": [], "return_annotation": "list[str]", "docstring": "Return names of all registered tool parsers.", "summary": "Return names of all registered tool parsers.", "implementation": "Method `ToolParserManager.list_registered` calls `sorted`, `set`, `cls.tool_parsers.keys`, `cls.lazy_parsers.keys`; returns `sorted(set(cls.tool_parsers.keys()) | set(cls.lazy_parsers.keys()))`.", "documented": true, "public": true, "addressable": true, "line": 284, "end_line": 286, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/abstract_tool_parser.py#L284-L286", "decorators": [ "classmethod" ], "calls": [ "sorted", "set", "cls.tool_parsers.keys", "cls.lazy_parsers.keys" ], "state_reads": [ "cls.tool_parsers.keys", "cls.tool_parsers", "cls.lazy_parsers.keys", "cls.lazy_parsers" ], "state_writes": [], "raises": [], "return_expressions": [ "sorted(set(cls.tool_parsers.keys()) | set(cls.lazy_parsers.keys()))" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.auto_tool_parser", "path": "vllm_mlx/tool_parsers/auto_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/auto_tool_parser.md", "docstring": "Auto-detecting tool call parser for vllm-mlx.\n\nAutomatically detects and parses tool calls from various model formats.", "summary": "Auto-detecting tool call parser for vllm-mlx.", "line_count": 414, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/auto_tool_parser.py#L1-L414", "members": [ "generate_tool_id", "AutoToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.auto_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/auto_tool_parser.py#L22-L24", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "AutoToolParser", "qualname": "AutoToolParser", "full_name": "vllm_mlx.tool_parsers.auto_tool_parser.AutoToolParser", "kind": "class", "signature": "class AutoToolParser(ToolParser)", "parameters": [], "return_annotation": "AutoToolParser", "docstring": "Auto-detecting tool call parser.\n\nTries multiple formats in order:\n1. Gemma 4: <|tool_call>call:name{...}\n2. Mistral: [TOOL_CALLS] ...\n3. Qwen bracket: [Calling tool: func_name({...})]\n4. Qwen/Hermes XML: {\"name\": \"...\", \"arguments\": {...}}\n5. Llama: {\"arg\": \"value\"}\n6. Nemotron: ...\n7. Raw JSON: {\"name\": \"...\", \"arguments\": {...}}\n\nThis is the default parser when no specific parser is selected.", "summary": "Auto-detecting tool call parser.", "implementation": "Class `AutoToolParser` derives from `ToolParser` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 414, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/auto_tool_parser.py#L28-L414", "decorators": [ "ToolParserManager.register_module(['auto', 'generic'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "AutoToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.auto_tool_parser.AutoToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls by trying all known formats.", "summary": "Extract tool calls by trying all known formats.", "implementation": "Method `AutoToolParser.extract_tool_calls` calls `Gemma4ToolParser`, `gemma_parser.extract_tool_calls`, `model_output.split`, `parts[0].strip`; has 4 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 61, "end_line": 268, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/auto_tool_parser.py#L61-L268", "decorators": [], "calls": [ "Gemma4ToolParser", "gemma_parser.extract_tool_calls", "model_output.split", "parts[0].strip", "raw.strip", "raw.startswith", "raw.find", "raw[:end_name].strip", "tool_calls.append", "generate_tool_id", "json.loads", "isinstance", "item.get", "json.dumps", "str", "ExtractedToolCallInformation", "self.QWEN_BRACKET_PATTERN.findall", "name.strip", "self.QWEN_BRACKET_PATTERN.sub('', cleaned_text).strip", "self.QWEN_BRACKET_PATTERN.sub", "self.BARE_BRACKET_PATTERN.findall", "self.BARE_BRACKET_PATTERN.sub('', cleaned_text).strip", "self.BARE_BRACKET_PATTERN.sub", "self.NEMOTRON_PATTERN.findall", "self.NEMOTRON_PARAM_PATTERN.findall", "p_name.strip", "p_value.strip", "self.NEMOTRON_PATTERN.sub('', cleaned_text).strip", "self.NEMOTRON_PATTERN.sub", "self.QWEN_XML_PATTERN.findall", "data.get", "self.QWEN_XML_PATTERN.sub('', cleaned_text).strip", "self.QWEN_XML_PATTERN.sub", "self.LLAMA_PATTERN.findall", "self.LLAMA_PATTERN.sub('', cleaned_text).strip", "self.LLAMA_PATTERN.sub", "self._parse_raw_json_tool_calls", "tool_calls.extend" ], "state_reads": [ "self.MISTRAL_TOKEN", "self.QWEN_BRACKET_PATTERN.findall", "self.QWEN_BRACKET_PATTERN", "self.QWEN_BRACKET_PATTERN.sub", "self.BARE_BRACKET_PATTERN.findall", "self.BARE_BRACKET_PATTERN", "self.BARE_BRACKET_PATTERN.sub", "self.NEMOTRON_PATTERN.findall", "self.NEMOTRON_PATTERN", "self.NEMOTRON_PARAM_PATTERN.findall", "self.NEMOTRON_PARAM_PATTERN", "self.NEMOTRON_PATTERN.sub", "self.QWEN_XML_PATTERN.findall", "self.QWEN_XML_PATTERN", "self.QWEN_XML_PATTERN.sub", "self.LLAMA_PATTERN.findall", "self.LLAMA_PATTERN", "self.LLAMA_PATTERN.sub", "self._parse_raw_json_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "result", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content if content else None)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=cleaned_text if cleaned_text else None)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)" ], "awaits": false, "yields": false }, { "name": "_parse_raw_json_tool_calls", "qualname": "AutoToolParser._parse_raw_json_tool_calls", "full_name": "vllm_mlx.tool_parsers.auto_tool_parser.AutoToolParser._parse_raw_json_tool_calls", "kind": "method", "signature": "def _parse_raw_json_tool_calls(self, text: str) -> list[dict[str, Any]]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict[str, Any]]", "docstring": "Parse raw JSON tool calls from text.\n\nHandles:\n- Single JSON object: {\"name\": \"func\", \"arguments\": {...}}\n- JSON array: [{...}, {...}]", "summary": "Parse raw JSON tool calls from text.", "implementation": "Method `AutoToolParser._parse_raw_json_tool_calls` calls `text.strip`, `text.startswith`, `json.loads`, `isinstance`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 270, "end_line": 350, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/auto_tool_parser.py#L270-L350", "decorators": [], "calls": [ "text.strip", "text.startswith", "json.loads", "isinstance", "item.get", "tool_calls.append", "generate_tool_id", "json.dumps", "str", "enumerate", "obj.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "tool_calls" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "AutoToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.auto_tool_parser.AutoToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming model output.\n\nUses simple heuristics to detect when a tool call might be complete.", "summary": "Extract tool calls from streaming model output.", "implementation": "Method `AutoToolParser.extract_tool_calls_streaming` calls `any`, `self.BARE_BRACKET_PARTIAL_PATTERN.search`, `self.BARE_BRACKET_PATTERN.search`, `self.extract_tool_calls`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 352, "end_line": 414, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/auto_tool_parser.py#L352-L414", "decorators": [], "calls": [ "any", "self.BARE_BRACKET_PARTIAL_PATTERN.search", "self.BARE_BRACKET_PATTERN.search", "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self.MISTRAL_TOKEN", "self.BARE_BRACKET_PARTIAL_PATTERN.search", "self.BARE_BRACKET_PARTIAL_PATTERN", "self.BARE_BRACKET_PATTERN.search", "self.BARE_BRACKET_PATTERN", "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.deepseek_tool_parser", "path": "vllm_mlx/tool_parsers/deepseek_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/deepseek_tool_parser.md", "docstring": "DeepSeek tool call parser for vllm-mlx.\n\nHandles DeepSeek V3 and R1 tool calling formats:\n- <｜tool▁calls▁begin｜>...<｜tool▁calls▁end｜> wrapper\n- <｜tool▁call▁begin｜>function<｜tool▁sep｜>name\n ```json\n {...}\n ```<｜tool▁call▁end｜>", "summary": "DeepSeek tool call parser for vllm-mlx.", "line_count": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/deepseek_tool_parser.py#L1-L170", "members": [ "generate_tool_id", "DeepSeekToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.deepseek_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 26, "end_line": 28, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/deepseek_tool_parser.py#L26-L28", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "DeepSeekToolParser", "qualname": "DeepSeekToolParser", "full_name": "vllm_mlx.tool_parsers.deepseek_tool_parser.DeepSeekToolParser", "kind": "class", "signature": "class DeepSeekToolParser(ToolParser)", "parameters": [], "return_annotation": "DeepSeekToolParser", "docstring": "Tool call parser for DeepSeek V3 and R1 models.\n\nSupports DeepSeek's tool call format with special unicode tokens:\n<｜tool▁calls▁begin｜>\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather\n```json\n{\"city\": \"Paris\"}\n```<｜tool▁call▁end｜>\n<｜tool▁calls▁end｜>\n\nUsed when --enable-auto-tool-choice --tool-call-parser deepseek are set.", "summary": "Tool call parser for DeepSeek V3 and R1 models.", "implementation": "Class `DeepSeekToolParser` derives from `ToolParser` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 32, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/deepseek_tool_parser.py#L32-L170", "decorators": [ "ToolParserManager.register_module(['deepseek', 'deepseek_v3', 'deepseek_r1'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "DeepSeekToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.deepseek_tool_parser.DeepSeekToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from DeepSeek model output.", "summary": "Extract tool calls from DeepSeek model output.", "implementation": "Method `DeepSeekToolParser.extract_tool_calls` calls `ExtractedToolCallInformation`, `model_output.find`, `model_output[:content_end].strip`, `self.TOOL_CALL_PATTERN.findall`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 69, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/deepseek_tool_parser.py#L69-L133", "decorators": [], "calls": [ "ExtractedToolCallInformation", "model_output.find", "model_output[:content_end].strip", "self.TOOL_CALL_PATTERN.findall", "json.loads", "tool_calls.append", "generate_tool_id", "func_name.strip", "func_args.strip", "self.TOOL_CALL_SIMPLE_PATTERN.findall" ], "state_reads": [ "self.TOOL_CALLS_START", "self.TOOL_CALL_PATTERN.findall", "self.TOOL_CALL_PATTERN", "self.TOOL_CALL_SIMPLE_PATTERN.findall", "self.TOOL_CALL_SIMPLE_PATTERN" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "DeepSeekToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.deepseek_tool_parser.DeepSeekToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming DeepSeek model output.", "summary": "Extract tool calls from streaming DeepSeek model output.", "implementation": "Method `DeepSeekToolParser.extract_tool_calls_streaming` calls `self.extract_tool_calls`, `enumerate`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 135, "end_line": 170, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/deepseek_tool_parser.py#L135-L170", "decorators": [], "calls": [ "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self.TOOL_CALLS_START", "self.TOOL_CALL_END", "self.TOOL_CALLS_END", "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.functionary_tool_parser", "path": "vllm_mlx/tool_parsers/functionary_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/functionary_tool_parser.md", "docstring": "Functionary tool call parser for vllm-mlx.\n\nHandles MeetKai Functionary models' tool calling format.\nSimilar to OpenAI function calling with JSON arguments.", "summary": "Functionary tool call parser for vllm-mlx.", "line_count": 193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/functionary_tool_parser.py#L1-L193", "members": [ "generate_tool_id", "FunctionaryToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.functionary_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/functionary_tool_parser.py#L22-L24", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "FunctionaryToolParser", "qualname": "FunctionaryToolParser", "full_name": "vllm_mlx.tool_parsers.functionary_tool_parser.FunctionaryToolParser", "kind": "class", "signature": "class FunctionaryToolParser(ToolParser)", "parameters": [], "return_annotation": "FunctionaryToolParser", "docstring": " Tool call parser for MeetKai Functionary models.\n\n Supports Functionary's tool call format similar to OpenAI:\n - Uses special tokens to mark tool calls\n - Arguments are JSON strings\n\n Formats supported:\n - <|from|>assistant\n<|recipient|>func_name\n<|content|>{\"args\": ...}\n - {\"args\": ...}\n\n Used when --enable-auto-tool-choice --tool-call-parser functionary are set.\n ", "summary": "Tool call parser for MeetKai Functionary models.", "implementation": "Class `FunctionaryToolParser` derives from `ToolParser` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/functionary_tool_parser.py#L28-L193", "decorators": [ "ToolParserManager.register_module(['functionary', 'meetkai'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "FunctionaryToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.functionary_tool_parser.FunctionaryToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from Functionary model output.", "summary": "Extract tool calls from Functionary model output.", "implementation": "Method `FunctionaryToolParser.extract_tool_calls` calls `self.RECIPIENT_PATTERN.findall`, `func_name.lower`, `json.loads`, `tool_calls.append`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 61, "end_line": 153, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/functionary_tool_parser.py#L61-L153", "decorators": [], "calls": [ "self.RECIPIENT_PATTERN.findall", "func_name.lower", "json.loads", "tool_calls.append", "generate_tool_id", "self.RECIPIENT_PATTERN.sub", "re.sub('<\\\\|from\\\\|>assistant\\\\s*', '', cleaned_text).strip", "re.sub", "self.FUNCTION_PATTERN.findall", "func_name.strip", "self.FUNCTION_PATTERN.sub('', cleaned_text).strip", "self.FUNCTION_PATTERN.sub", "self.JSON_ARRAY_PATTERN.match", "model_output.strip", "isinstance", "call.get", "json.dumps", "str", "ExtractedToolCallInformation" ], "state_reads": [ "self.RECIPIENT_PATTERN.findall", "self.RECIPIENT_PATTERN", "self.RECIPIENT_PATTERN.sub", "self.FUNCTION_PATTERN.findall", "self.FUNCTION_PATTERN", "self.FUNCTION_PATTERN.sub", "self.JSON_ARRAY_PATTERN.match", "self.JSON_ARRAY_PATTERN" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=cleaned_text if cleaned_text else None)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "FunctionaryToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.functionary_tool_parser.FunctionaryToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Functionary model output.", "summary": "Extract tool calls from streaming Functionary model output.", "implementation": "Method `FunctionaryToolParser.extract_tool_calls_streaming` calls `any`, `self.extract_tool_calls`, `enumerate`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 155, "end_line": 193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/functionary_tool_parser.py#L155-L193", "decorators": [], "calls": [ "any", "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.gemma4_tool_parser", "path": "vllm_mlx/tool_parsers/gemma4_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/gemma4_tool_parser.md", "docstring": "Gemma 4 tool call parser for vllm-mlx.\n\nHandles Gemma 4's native tool call format:\n <|tool_call>call:func_name{<|\"|>key<|\"|>: <|\"|>value<|\"|>, num: 42}\n\nGemma 4 uses special tokens instead of JSON:\n- <|tool_call> / delimit tool call blocks\n- <|\"|> replaces \" for string values\n- Keys are unquoted bare identifiers\n- Multiple call:name{...} can appear in a single block\n\nFallback forms (issue #80): under long system prompts + multi-turn + several\ntools, Gemma 4 frequently abandons the canonical brace form and instead emits\nits call as plain text in `content`, using Python-style call syntax:\n\n e4b: <|tool_call>call:radarr_get_movies(search=\"Dune\")\n e2b: ```tool_code\n radarr_get_movies(search=\"Dune\")\n ```\n e2b: tool_code = radarr_get_movies(search=\"Dune\")\n print(tool_code)\n\nThese are parsed by a fallback layer (ast-based) when the canonical parse finds\nno calls, so the host can still dispatch the tool.\n\nReference: mlx-lm PR #1105, vllm PR #38837", "summary": "Gemma 4 tool call parser for vllm-mlx.", "line_count": 513, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L1-L513", "members": [ "logger", "TOOL_CALL_START", "TOOL_CALL_END", "_PLACEHOLDER_RE", "_STRING_DELIM_RE", "_CALL_PREFIX", "_BARE_KEY", "_BARE_VALUE", "_JSON_LITERALS", "_MAX_ARG_BLOCK_LEN", "_CALL_PAREN_RE", "_TOOL_CODE_FENCE_RE", "_TOOL_CODE_ASSIGN_RE", "_FALLBACK_MARKER_RE", "_find_balanced_brace", "_find_balanced_paren", "_quote_bare_value", "_gemma4_args_to_json", "_call_node_to_tool", "_parse_python_call", "_parse_calls_from_code", "_strip_spans", "generate_tool_id", "Gemma4ToolParser" ], "symbols": [ { "name": "_find_balanced_brace", "qualname": "_find_balanced_brace", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._find_balanced_brace", "kind": "function", "signature": "def _find_balanced_brace(text: str, start: int) -> int", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The string to search (may contain <|\"|> tokens)" }, { "name": "start", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Index of the opening {" } ], "return_annotation": "int", "docstring": "Find the index of the closing } that balances the { at `start`.\n\nBefore counting braces, <|\"|>-delimited strings are conceptually opaque --\nwe skip over <|\"|>...<|\"|> regions so that braces inside string values\n(e.g. code snippets) don't affect depth counting.\n\nArgs:\n text: The string to search (may contain <|\"|> tokens)\n start: Index of the opening {\n\nReturns:\n Index of the matching } in the ORIGINAL text, or -1 if not found", "summary": "Find the index of the closing } that balances the { at `start`.", "implementation": "Function `_find_balanced_brace` calls `len`, `text.startswith`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 92, "end_line": 125, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L92-L125", "decorators": [], "calls": [ "len", "text.startswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "-1", "i" ], "awaits": false, "yields": false }, { "name": "_find_balanced_paren", "qualname": "_find_balanced_paren", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._find_balanced_paren", "kind": "function", "signature": "def _find_balanced_paren(text: str, start: int) -> int", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "start", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Find the index of the closing ) that balances the ( at `start`.\n\nPython string literals ('...'/\"...\") are treated as opaque so that parens\ninside string argument values don't affect depth counting.\n\nReturns the index of the matching ) in `text`, or -1 if not found.", "summary": "Find the index of the closing ) that balances the ( at `start`.", "implementation": "Function `_find_balanced_paren` calls `len`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 128, "end_line": 159, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L128-L159", "decorators": [], "calls": [ "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "-1", "i" ], "awaits": false, "yields": false }, { "name": "_quote_bare_value", "qualname": "_quote_bare_value", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._quote_bare_value", "kind": "function", "signature": "def _quote_bare_value(m: re.Match) -> str", "parameters": [ { "name": "m", "kind": "positional or keyword", "annotation": "re.Match", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Substitution callback for _BARE_VALUE — quotes bare identifiers that\nare not JSON literals (true/false/null).", "summary": "Substitution callback for _BARE_VALUE — quotes bare identifiers that are not JSON literals (true/false/null).", "implementation": "Function `_quote_bare_value` calls `m.group`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 162, "end_line": 168, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L162-L168", "decorators": [], "calls": [ "m.group" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "m.group(0)", "f'{ws}\"{word}\"'" ], "awaits": false, "yields": false }, { "name": "_gemma4_args_to_json", "qualname": "_gemma4_args_to_json", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._gemma4_args_to_json", "kind": "function", "signature": "def _gemma4_args_to_json(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Convert Gemma 4 tool call args to valid JSON.\n\nFour-step conversion (ORDER MATTERS):\n1. Extract <|\"|>-delimited strings into numbered \\x00N\\x00 placeholders.\n This protects string contents from step 2's bare-key quoting -- without\n this, a string value like \"key: value\" would be corrupted.\n2. Quote bare keys (word: -> \"word\":) now that strings are safe.\n3. Quote bare string VALUES that the template emitted without <|\"|>\n wrappers. Happens with nullable/enum schemas where the STRING branch\n of the template isn't taken.\n4. Restore placeholders as properly JSON-escaped strings via json.dumps().\n Uses a single re.sub pass (O(len(text))) instead of per-placeholder replace.", "summary": "Convert Gemma 4 tool call args to valid JSON.", "implementation": "Function `_gemma4_args_to_json` calls `_STRING_DELIM_RE.sub`, `_BARE_KEY.sub`, `_BARE_VALUE.sub`, `_PLACEHOLDER_RE.sub`; returns `text`.", "documented": true, "public": false, "addressable": true, "line": 171, "end_line": 207, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L171-L207", "decorators": [], "calls": [ "_STRING_DELIM_RE.sub", "_BARE_KEY.sub", "_BARE_VALUE.sub", "_PLACEHOLDER_RE.sub" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text" ], "awaits": false, "yields": false }, { "name": "_capture", "qualname": "_gemma4_args_to_json._capture", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._gemma4_args_to_json._capture", "kind": "nested function", "signature": "def _capture(m: re.Match) -> str", "parameters": [ { "name": "m", "kind": "positional or keyword", "annotation": "re.Match", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Nested Function `_gemma4_args_to_json._capture` calls `strings.append`, `m.group`, `len`; returns `f'\\x00{len(strings) - 1}\\x00'`.", "implementation": "Nested Function `_gemma4_args_to_json._capture` calls `strings.append`, `m.group`, `len`; returns `f'\\x00{len(strings) - 1}\\x00'`.", "documented": false, "public": false, "addressable": false, "line": 187, "end_line": 189, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L187-L189", "decorators": [], "calls": [ "strings.append", "m.group", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'\\x00{len(strings) - 1}\\x00'" ], "awaits": false, "yields": false }, { "name": "_restore", "qualname": "_gemma4_args_to_json._restore", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._gemma4_args_to_json._restore", "kind": "nested function", "signature": "def _restore(m: re.Match) -> str", "parameters": [ { "name": "m", "kind": "positional or keyword", "annotation": "re.Match", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Nested Function `_gemma4_args_to_json._restore` calls `int`, `m.group`, `len`, `json.dumps`; returns `json.dumps(strings[idx]) if idx < len(strings) else m.group(0)`.", "implementation": "Nested Function `_gemma4_args_to_json._restore` calls `int`, `m.group`, `len`, `json.dumps`; returns `json.dumps(strings[idx]) if idx < len(strings) else m.group(0)`.", "documented": false, "public": false, "addressable": false, "line": 201, "end_line": 203, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L201-L203", "decorators": [], "calls": [ "int", "m.group", "len", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.dumps(strings[idx]) if idx < len(strings) else m.group(0)" ], "awaits": false, "yields": false }, { "name": "_call_node_to_tool", "qualname": "_call_node_to_tool", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._call_node_to_tool", "kind": "function", "signature": "def _call_node_to_tool(call: ast.Call) -> tuple[str, dict[str, Any]] | None", "parameters": [ { "name": "call", "kind": "positional or keyword", "annotation": "ast.Call", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, dict[str, Any]] | None", "docstring": "Map a Python `ast.Call` node to (function_name, kwargs_dict).\n\nOnly keyword arguments are mapped (Gemma emits its tool calls as kwargs);\npositional args are ignored because the parameter names aren't recoverable.\nReturns None if the name or any argument value isn't a plain literal.", "summary": "Map a Python `ast.Call` node to (function_name, kwargs_dict).", "implementation": "Function `_call_node_to_tool` calls `isinstance`, `ast.literal_eval`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 210, "end_line": 233, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L210-L233", "decorators": [], "calls": [ "isinstance", "ast.literal_eval" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "(name, args)" ], "awaits": false, "yields": false }, { "name": "_parse_python_call", "qualname": "_parse_python_call", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._parse_python_call", "kind": "function", "signature": "def _parse_python_call(src: str) -> tuple[str, dict[str, Any]] | None", "parameters": [ { "name": "src", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, dict[str, Any]] | None", "docstring": "Parse a single `fn(...)` Python call expression into (name, kwargs).", "summary": "Parse a single `fn(...)` Python call expression into (name, kwargs).", "implementation": "Function `_parse_python_call` calls `ast.parse`, `src.strip`, `isinstance`, `_call_node_to_tool`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 236, "end_line": 244, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L236-L244", "decorators": [], "calls": [ "ast.parse", "src.strip", "isinstance", "_call_node_to_tool" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "_call_node_to_tool(node.body)" ], "awaits": false, "yields": false }, { "name": "_parse_calls_from_code", "qualname": "_parse_calls_from_code", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._parse_calls_from_code", "kind": "function", "signature": "def _parse_calls_from_code(code: str) -> list[tuple[str, dict[str, Any]]]", "parameters": [ { "name": "code", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[tuple[str, dict[str, Any]]]", "docstring": "Parse every top-level `fn(...)` call statement in a code-fence body.", "summary": "Parse every top-level `fn(...)` call statement in a code-fence body.", "implementation": "Function `_parse_calls_from_code` calls `ast.parse`, `textwrap.dedent(code).strip`, `textwrap.dedent`, `isinstance`; returns `results`.", "documented": true, "public": false, "addressable": true, "line": 247, "end_line": 259, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L247-L259", "decorators": [], "calls": [ "ast.parse", "textwrap.dedent(code).strip", "textwrap.dedent", "isinstance", "_call_node_to_tool", "results.append" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "results" ], "awaits": false, "yields": false }, { "name": "_strip_spans", "qualname": "_strip_spans", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser._strip_spans", "kind": "function", "signature": "def _strip_spans(text: str, spans: list[tuple[int, int]]) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "spans", "kind": "positional or keyword", "annotation": "list[tuple[int, int]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Remove the given [start, end) spans from `text` (handles overlaps).", "summary": "Remove the given [start, end) spans from `text` (handles overlaps).", "implementation": "Function `_strip_spans` calls `sorted`, `max`, `out.append`, `''.join`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 262, "end_line": 275, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L262-L275", "decorators": [], "calls": [ "sorted", "max", "out.append", "''.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text", "''.join(out)" ], "awaits": false, "yields": false }, { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 278, "end_line": 280, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L278-L280", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "Gemma4ToolParser", "qualname": "Gemma4ToolParser", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser.Gemma4ToolParser", "kind": "class", "signature": "class Gemma4ToolParser(ToolParser)", "parameters": [], "return_annotation": "Gemma4ToolParser", "docstring": "Tool call parser for Gemma 4 models.\n\nParses: <|tool_call>call:func{<|\"|>key<|\"|>: <|\"|>val<|\"|>}\n\nUsed when --enable-auto-tool-choice --tool-call-parser gemma4 are set.", "summary": "Tool call parser for Gemma 4 models.", "implementation": "Class `Gemma4ToolParser` derives from `ToolParser` and declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 284, "end_line": 513, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L284-L513", "decorators": [ "ToolParserManager.register_module('gemma4')" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "Gemma4ToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser.Gemma4ToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete Gemma 4 model response.", "summary": "Extract tool calls from a complete Gemma 4 model response.", "implementation": "Method `Gemma4ToolParser.extract_tool_calls` calls `self.strip_think_tags`, `self._extract_canonical`, `ExtractedToolCallInformation`, `self._extract_fallback`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 300, "end_line": 324, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L300-L324", "decorators": [], "calls": [ "self.strip_think_tags", "self._extract_canonical", "ExtractedToolCallInformation", "self._extract_fallback" ], "state_reads": [ "self.strip_think_tags", "self._extract_canonical", "self._extract_fallback" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content_before)", "fallback", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)" ], "awaits": false, "yields": false }, { "name": "_extract_canonical", "qualname": "Gemma4ToolParser._extract_canonical", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser.Gemma4ToolParser._extract_canonical", "kind": "method", "signature": "def _extract_canonical(self, cleaned: str) -> tuple[list[dict[str, Any]], str | None]", "parameters": [ { "name": "cleaned", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[list[dict[str, Any]], str | None]", "docstring": "Parse the canonical <|tool_call>call:fn{...} form.\n\nReturns (tool_calls, content_before). tool_calls is empty when the\ncanonical markers/braces aren't present.", "summary": "Parse the canonical <|tool_call>call:fn{...} form.", "implementation": "Method `Gemma4ToolParser._extract_canonical` calls `cleaned.find`, `cleaned[:start_idx].strip`, `len`, `_CALL_PREFIX.search`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 326, "end_line": 382, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L326-L382", "decorators": [], "calls": [ "cleaned.find", "cleaned[:start_idx].strip", "len", "_CALL_PREFIX.search", "m.group", "m.end", "_find_balanced_brace", "_gemma4_args_to_json", "json.loads", "tool_calls.append", "generate_tool_id", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "([], None)", "(tool_calls, content_before)" ], "awaits": false, "yields": false }, { "name": "_extract_fallback", "qualname": "Gemma4ToolParser._extract_fallback", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser.Gemma4ToolParser._extract_fallback", "kind": "method", "signature": "def _extract_fallback(self, cleaned: str) -> ExtractedToolCallInformation | None", "parameters": [ { "name": "cleaned", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "ExtractedToolCallInformation | None", "docstring": "Parse the Python-style fallback forms (issue #80).\n\nHandles ```tool_code``` blocks (bare `fn(...)` calls) and the\nparenthesized `call:fn(...)` form. Returns None if neither is present.", "summary": "Parse the Python-style fallback forms (issue #80).", "implementation": "Method `Gemma4ToolParser._extract_fallback` calls `_TOOL_CODE_FENCE_RE.finditer`, `_parse_calls_from_code`, `m.group`, `tool_calls.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 384, "end_line": 463, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L384-L463", "decorators": [], "calls": [ "_TOOL_CODE_FENCE_RE.finditer", "_parse_calls_from_code", "m.group", "tool_calls.append", "generate_tool_id", "json.dumps", "spans.append", "m.start", "m.end", "_CALL_PAREN_RE.finditer", "any", "_find_balanced_paren", "_parse_python_call", "_TOOL_CODE_ASSIGN_RE.finditer", "_strip_spans", "content.replace(TOOL_CALL_START, '').replace", "content.replace", "re.sub", "content.strip", "ExtractedToolCallInformation" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content)" ], "awaits": false, "yields": false }, { "name": "_format_streaming", "qualname": "Gemma4ToolParser._format_streaming", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser.Gemma4ToolParser._format_streaming", "kind": "method", "signature": "def _format_streaming(self, result: ExtractedToolCallInformation) -> dict[str, Any]", "parameters": [ { "name": "result", "kind": "positional or keyword", "annotation": "ExtractedToolCallInformation", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, Any]", "docstring": "Render extracted tool calls into the streaming delta shape.", "summary": "Render extracted tool calls into the streaming delta shape.", "implementation": "Method `Gemma4ToolParser._format_streaming` calls `enumerate`; returns `{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…`.", "documented": true, "public": false, "addressable": true, "line": 465, "end_line": 480, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L465-L480", "decorators": [], "calls": [ "enumerate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "Gemma4ToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.gemma4_tool_parser.Gemma4ToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Gemma 4 model output.", "summary": "Extract tool calls from streaming Gemma 4 model output.", "implementation": "Method `Gemma4ToolParser.extract_tool_calls_streaming` calls `bool`, `_FALLBACK_MARKER_RE.search`, `self.extract_tool_calls`, `self._format_streaming`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 482, "end_line": 513, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/gemma4_tool_parser.py#L482-L513", "decorators": [], "calls": [ "bool", "_FALLBACK_MARKER_RE.search", "self.extract_tool_calls", "self._format_streaming" ], "state_reads": [ "self.extract_tool_calls", "self._format_streaming" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "self._format_streaming(result)", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.glm47_tool_parser", "path": "vllm_mlx/tool_parsers/glm47_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/glm47_tool_parser.md", "docstring": "GLM-4.7 tool call parser for vllm-mlx.\n\nHandles GLM-4.7-Flash style tool calling format.\nBased on vLLM's glm47_moe_tool_parser.py", "summary": "GLM-4.7 tool call parser for vllm-mlx.", "line_count": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/glm47_tool_parser.py#L1-L184", "members": [ "generate_tool_id", "Glm47ToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.glm47_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/glm47_tool_parser.py#L22-L24", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "Glm47ToolParser", "qualname": "Glm47ToolParser", "full_name": "vllm_mlx.tool_parsers.glm47_tool_parser.Glm47ToolParser", "kind": "class", "signature": "class Glm47ToolParser(ToolParser)", "parameters": [], "return_annotation": "Glm47ToolParser", "docstring": "Tool call parser for GLM-4.7 and GLM-4.7-Flash models.\n\nSupports GLM-4.7 tool call format:\nfunction_name\nparam1value1\nparam2value2\n\n\nUsed when --enable-auto-tool-choice --tool-call-parser glm47 are set.", "summary": "Tool call parser for GLM-4.7 and GLM-4.7-Flash models.", "implementation": "Class `Glm47ToolParser` derives from `ToolParser` and declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/glm47_tool_parser.py#L28-L184", "decorators": [ "ToolParserManager.register_module(['glm47', 'glm4'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_deserialize", "qualname": "Glm47ToolParser._deserialize", "full_name": "vllm_mlx.tool_parsers.glm47_tool_parser.Glm47ToolParser._deserialize", "kind": "method", "signature": "def _deserialize(self, value: str) -> Any", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Convert string value to appropriate Python type.\n\nUses json.loads for type coercion, falls back to raw string.", "summary": "Convert string value to appropriate Python type.", "implementation": "Method `Glm47ToolParser._deserialize` calls `value.strip`, `json.loads`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 57, "end_line": 67, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/glm47_tool_parser.py#L57-L67", "decorators": [], "calls": [ "value.strip", "json.loads" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.loads(value)", "value" ], "awaits": false, "yields": false }, { "name": "_get_tool_names", "qualname": "Glm47ToolParser._get_tool_names", "full_name": "vllm_mlx.tool_parsers.glm47_tool_parser.Glm47ToolParser._get_tool_names", "kind": "method", "signature": "def _get_tool_names(self, request: dict[str, Any] | None) -> set[str]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "set[str]", "docstring": "Extract valid tool names from the request.", "summary": "Extract valid tool names from the request.", "implementation": "Method `Glm47ToolParser._get_tool_names` calls `set`, `t.get('function', {}).get`, `t.get`, `request.get`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 69, "end_line": 77, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/glm47_tool_parser.py#L69-L77", "decorators": [], "calls": [ "set", "t.get('function', {}).get", "t.get", "request.get", "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "set()", "{t.get('function', {}).get('name', '') for t in request.get('tools', []) if isinstance(t, dict)}" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "Glm47ToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.glm47_tool_parser.Glm47ToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete GLM-4.7 model response.", "summary": "Extract tool calls from a complete GLM-4.7 model response.", "implementation": "Method `Glm47ToolParser.extract_tool_calls` calls `self.strip_think_tags`, `self._get_tool_names`, `self.FUNC_DETAIL_PATTERN.findall`, `match[0].strip`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 79, "end_line": 137, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/glm47_tool_parser.py#L79-L137", "decorators": [], "calls": [ "self.strip_think_tags", "self._get_tool_names", "self.FUNC_DETAIL_PATTERN.findall", "match[0].strip", "len", "self.ARG_PATTERN.findall", "arg_key.strip", "self._deserialize", "tool_calls.append", "generate_tool_id", "json.dumps", "ExtractedToolCallInformation" ], "state_reads": [ "self.strip_think_tags", "self._get_tool_names", "self.FUNC_DETAIL_PATTERN.findall", "self.FUNC_DETAIL_PATTERN", "self.ARG_PATTERN.findall", "self.ARG_PATTERN", "self._deserialize" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=None)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=cleaned_text)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "Glm47ToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.glm47_tool_parser.Glm47ToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming GLM-4.7 model output.", "summary": "Extract tool calls from streaming GLM-4.7 model output.", "implementation": "Method `Glm47ToolParser.extract_tool_calls_streaming` calls `self.extract_tool_calls`, `enumerate`, `self.strip_think_tags`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 139, "end_line": 184, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/glm47_tool_parser.py#L139-L184", "decorators": [], "calls": [ "self.extract_tool_calls", "enumerate", "self.strip_think_tags" ], "state_reads": [ "self.extract_tool_calls", "self.strip_think_tags" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "{'content': clean_delta}" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.granite_tool_parser", "path": "vllm_mlx/tool_parsers/granite_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/granite_tool_parser.md", "docstring": "Granite tool call parser for vllm-mlx.\n\nHandles IBM Granite models' tool calling format:\n- <|tool_call|> or followed by JSON array\n- [{\"name\": \"func\", \"arguments\": {...}}]", "summary": "Granite tool call parser for vllm-mlx.", "line_count": 147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/granite_tool_parser.py#L1-L147", "members": [ "generate_tool_id", "GraniteToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.granite_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/granite_tool_parser.py#L22-L24", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "GraniteToolParser", "qualname": "GraniteToolParser", "full_name": "vllm_mlx.tool_parsers.granite_tool_parser.GraniteToolParser", "kind": "class", "signature": "class GraniteToolParser(ToolParser)", "parameters": [], "return_annotation": "GraniteToolParser", "docstring": "Tool call parser for IBM Granite models.\n\nSupports Granite's tool call format:\n<|tool_call|>[{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}]\n\nOr Granite 3.1:\n[{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}]\n\nUsed when --enable-auto-tool-choice --tool-call-parser granite are set.", "summary": "Tool call parser for IBM Granite models.", "implementation": "Class `GraniteToolParser` derives from `ToolParser` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/granite_tool_parser.py#L28-L147", "decorators": [ "ToolParserManager.register_module(['granite', 'granite3'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "GraniteToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.granite_tool_parser.GraniteToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from Granite model output.", "summary": "Extract tool calls from Granite model output.", "implementation": "Method `GraniteToolParser.extract_tool_calls` calls `model_output.strip`, `stripped.startswith`, `stripped[len(self.BOT_TOKEN):].lstrip`, `len`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 47, "end_line": 105, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/granite_tool_parser.py#L47-L105", "decorators": [], "calls": [ "model_output.strip", "stripped.startswith", "stripped[len(self.BOT_TOKEN):].lstrip", "len", "stripped[len(self.BOT_STRING):].lstrip", "ExtractedToolCallInformation", "json.loads", "isinstance", "call.get", "tool_calls.append", "generate_tool_id", "json.dumps", "str" ], "state_reads": [ "self.BOT_TOKEN", "self.BOT_STRING" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=None)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "GraniteToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.granite_tool_parser.GraniteToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Granite model output.", "summary": "Extract tool calls from streaming Granite model output.", "implementation": "Method `GraniteToolParser.extract_tool_calls_streaming` calls `current_text.strip`, `stripped.startswith`, `self.extract_tool_calls`, `enumerate`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 107, "end_line": 147, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/granite_tool_parser.py#L107-L147", "decorators": [], "calls": [ "current_text.strip", "stripped.startswith", "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self.BOT_TOKEN", "self.BOT_STRING", "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.harmony_tool_parser", "path": "vllm_mlx/tool_parsers/harmony_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/harmony_tool_parser.md", "docstring": "Harmony tool call parser for GPT-OSS models.\n\nHarmony uses control tokens and channels for tool calling:\n\n <|channel|>commentary to=functions.get_weather\n <|constrain|>json\n <|message|>{\"location\": \"San Francisco\"}\n <|call|>\n\nThe final response is in the 'final' channel:\n\n <|channel|>final\n <|message|>The weather is 72F.\n <|return|>", "summary": "Harmony tool call parser for GPT-OSS models.", "line_count": 253, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L1-L253", "members": [ "_generate_tool_id", "_COMMENTARY_BLOCK_PATTERN", "_FINAL_BLOCK_PATTERN", "HarmonyToolParser", "_strip_control_tokens", "_is_control_token" ], "symbols": [ { "name": "_generate_tool_id", "qualname": "_generate_tool_id", "full_name": "vllm_mlx.tool_parsers.harmony_tool_parser._generate_tool_id", "kind": "function", "signature": "def _generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `_generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": false, "addressable": true, "line": 32, "end_line": 34, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L32-L34", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "HarmonyToolParser", "qualname": "HarmonyToolParser", "full_name": "vllm_mlx.tool_parsers.harmony_tool_parser.HarmonyToolParser", "kind": "class", "signature": "class HarmonyToolParser(ToolParser)", "parameters": [], "return_annotation": "HarmonyToolParser", "docstring": "Tool call parser for GPT-OSS models using Harmony format.\n\nHarmony uses control tokens and 3 channels:\n- analysis: internal reasoning (handled by reasoning parser)\n- commentary: tool calls addressed with to=functions.{name}\n- final: user-facing response\n\nUsed when --enable-auto-tool-choice --tool-call-parser harmony are set.", "summary": "Tool call parser for GPT-OSS models using Harmony format.", "implementation": "Class `HarmonyToolParser` derives from `ToolParser` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 57, "end_line": 219, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L57-L219", "decorators": [ "ToolParserManager.register_module(['harmony', 'gpt-oss'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "HarmonyToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.harmony_tool_parser.HarmonyToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete Harmony model response.\n\nParses commentary channel blocks for tool calls and the final\nchannel for the user-facing content.", "summary": "Extract tool calls from a complete Harmony model response.", "implementation": "Method `HarmonyToolParser.extract_tool_calls` calls `_COMMENTARY_BLOCK_PATTERN.finditer`, `match.group`, `match.group(2).strip`, `json.loads`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 71, "end_line": 140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L71-L140", "decorators": [], "calls": [ "_COMMENTARY_BLOCK_PATTERN.finditer", "match.group", "match.group(2).strip", "json.loads", "tool_calls.append", "_generate_tool_id", "isinstance", "json.dumps", "str", "_FINAL_BLOCK_PATTERN.search", "final_match.group(1).strip", "final_match.group", "ExtractedToolCallInformation", "_strip_control_tokens" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=content)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "HarmonyToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.harmony_tool_parser.HarmonyToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Harmony model output.\n\nA commentary block completes when an explicit terminator arrives\n(<|call|>, <|end|>, <|return|>, <|start|>) or when the model moves on\nto the <|channel|>final block; the completed call is emitted once\n(deduplicated by name + arguments). Final-channel content is emitted\nas regular content deltas and plain text passes through unchanged.", "summary": "Extract tool calls from streaming Harmony model output.", "implementation": "Method `HarmonyToolParser.extract_tool_calls_streaming` updates `self._emitted_streaming_signatures`; calls `hasattr`, `set`, `any`, `self.extract_tool_calls`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 142, "end_line": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L142-L214", "decorators": [], "calls": [ "hasattr", "set", "any", "self.extract_tool_calls", "enumerate", "self._emitted_streaming_signatures.add", "emitted.append", "current_text.rfind", "current_text.find", "len", "msg_content.replace('<|return|>', '').strip", "msg_content.replace", "_is_control_token" ], "state_reads": [ "self.extract_tool_calls", "self._emitted_streaming_signatures", "self._emitted_streaming_signatures.add" ], "state_writes": [ "self._emitted_streaming_signatures" ], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': emitted}", "None" ], "awaits": false, "yields": false }, { "name": "reset", "qualname": "HarmonyToolParser.reset", "full_name": "vllm_mlx.tool_parsers.harmony_tool_parser.HarmonyToolParser.reset", "kind": "method", "signature": "def reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset parser state for a new request.", "summary": "Reset parser state for a new request.", "implementation": "Method `HarmonyToolParser.reset` updates `self._emitted_streaming_signatures`; calls `super().reset`, `super`, `set`.", "documented": true, "public": true, "addressable": true, "line": 216, "end_line": 219, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L216-L219", "decorators": [], "calls": [ "super().reset", "super", "set" ], "state_reads": [], "state_writes": [ "self._emitted_streaming_signatures" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_strip_control_tokens", "qualname": "_strip_control_tokens", "full_name": "vllm_mlx.tool_parsers.harmony_tool_parser._strip_control_tokens", "kind": "function", "signature": "def _strip_control_tokens(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Remove Harmony control tokens from text.", "summary": "Remove Harmony control tokens from text.", "implementation": "Function `_strip_control_tokens` calls `result.replace`, `re.sub`, `result.strip`; returns `result.strip()`.", "documented": true, "public": false, "addressable": true, "line": 222, "end_line": 240, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L222-L240", "decorators": [], "calls": [ "result.replace", "re.sub", "result.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result.strip()" ], "awaits": false, "yields": false }, { "name": "_is_control_token", "qualname": "_is_control_token", "full_name": "vllm_mlx.tool_parsers.harmony_tool_parser._is_control_token", "kind": "function", "signature": "def _is_control_token(text: str) -> bool", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if text is a Harmony control token.", "summary": "Check if text is a Harmony control token.", "implementation": "Function `_is_control_token` calls `text.strip`; returns `text.strip() in {'<|start|>', '<|end|>', '<|message|>', '<|channel|>', '<|constrain|>', '<|return|>', '<|call|>'}`.", "documented": true, "public": false, "addressable": true, "line": 243, "end_line": 253, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/harmony_tool_parser.py#L243-L253", "decorators": [], "calls": [ "text.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text.strip() in {'<|start|>', '<|end|>', '<|message|>', '<|channel|>', '<|constrain|>', '<|return|>', '<|call|>'}" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.hermes_tool_parser", "path": "vllm_mlx/tool_parsers/hermes_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/hermes_tool_parser.md", "docstring": "Hermes/Nous tool call parser for vllm-mlx.\n\nHandles Hermes-style tool calling format used by NousResearch models.", "summary": "Hermes/Nous tool call parser for vllm-mlx.", "line_count": 336, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/hermes_tool_parser.py#L1-L336", "members": [ "generate_tool_id", "_parse_param_value", "HermesToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.hermes_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/hermes_tool_parser.py#L22-L24", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "_parse_param_value", "qualname": "_parse_param_value", "full_name": "vllm_mlx.tool_parsers.hermes_tool_parser._parse_param_value", "kind": "function", "signature": "def _parse_param_value(val: str) -> Any", "parameters": [ { "name": "val", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Parse a tool call parameter value, handling both JSON and Python literals.\n\nTries json.loads first. If that fails, falls back to ast.literal_eval\nfor Python literal syntax (single quotes, True/False, None). Converts\nsets to lists and rejects types that are not JSON-serializable (complex,\nbytes) to avoid crashes during json.dumps later.", "summary": "Parse a tool call parameter value, handling both JSON and Python literals.", "implementation": "Function `_parse_param_value` calls `json.loads`, `ast.literal_eval`, `isinstance`, `sorted`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 27, "end_line": 49, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/hermes_tool_parser.py#L27-L49", "decorators": [], "calls": [ "json.loads", "ast.literal_eval", "isinstance", "sorted", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.loads(val)", "val", "python_val" ], "awaits": false, "yields": false }, { "name": "HermesToolParser", "qualname": "HermesToolParser", "full_name": "vllm_mlx.tool_parsers.hermes_tool_parser.HermesToolParser", "kind": "class", "signature": "class HermesToolParser(ToolParser)", "parameters": [], "return_annotation": "HermesToolParser", "docstring": "Tool call parser for Hermes/Nous models.\n\nSupports Hermes tool call format:\n- {\"name\": \"func\", \"arguments\": {...}}\n- Sometimes with additional reasoning in \n- Fallback: raw JSON {\"name\": \"func\", \"arguments\": {...}} (for models that omit tags)\n\nUsed when --enable-auto-tool-choice --tool-call-parser hermes are set.", "summary": "Tool call parser for Hermes/Nous models.", "implementation": "Class `HermesToolParser` derives from `ToolParser` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 53, "end_line": 336, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/hermes_tool_parser.py#L53-L336", "decorators": [ "ToolParserManager.register_module(['hermes', 'nous'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "HermesToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.hermes_tool_parser.HermesToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete Hermes model response.", "summary": "Extract tool calls from a complete Hermes model response.", "implementation": "Method `HermesToolParser.extract_tool_calls` calls `self.strip_think_tags`, `self.REASONING_PATTERN.findall`, `self.REASONING_PATTERN.sub`, `self.TOOL_CALL_PATTERN.findall`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 92, "end_line": 245, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/hermes_tool_parser.py#L92-L245", "decorators": [], "calls": [ "self.strip_think_tags", "self.REASONING_PATTERN.findall", "self.REASONING_PATTERN.sub", "self.TOOL_CALL_PATTERN.findall", "json.loads", "data.get", "tool_calls.append", "generate_tool_id", "isinstance", "json.dumps", "str", "self.TOOL_CALL_PATTERN.sub('', cleaned_text).strip", "self.TOOL_CALL_PATTERN.sub", "self.NEMOTRON_PATTERN.findall", "self.PARAM_PATTERN.findall", "p_name.strip", "_parse_param_value", "p_value.strip", "name.strip", "self.NEMOTRON_PATTERN.sub('', cleaned_text).strip", "self.NEMOTRON_PATTERN.sub", "self.BARE_FUNCTION_PATTERN.findall", "self.BARE_FUNCTION_PATTERN.sub('', cleaned_text).strip", "self.BARE_FUNCTION_PATTERN.sub", "self.TOOL_CALL_LENIENT_PATTERN.findall", "self.TOOL_CALL_LENIENT_PATTERN.sub('', cleaned_text, count=1).strip", "self.TOOL_CALL_LENIENT_PATTERN.sub", "self.RAW_JSON_TOOL_PATTERN.findall", "t.get('function', {}).get", "t.get", "request.get", "self.RAW_JSON_TOOL_PATTERN.sub('', cleaned_text, count=1).strip", "self.RAW_JSON_TOOL_PATTERN.sub", "' '.join", "ExtractedToolCallInformation" ], "state_reads": [ "self.strip_think_tags", "self.REASONING_PATTERN.findall", "self.REASONING_PATTERN", "self.REASONING_PATTERN.sub", "self.TOOL_CALL_PATTERN.findall", "self.TOOL_CALL_PATTERN", "self.TOOL_CALL_PATTERN.sub", "self.NEMOTRON_PATTERN.findall", "self.NEMOTRON_PATTERN", "self.PARAM_PATTERN.findall", "self.PARAM_PATTERN", "self.NEMOTRON_PATTERN.sub", "self.BARE_FUNCTION_PATTERN.findall", "self.BARE_FUNCTION_PATTERN", "self.BARE_FUNCTION_PATTERN.sub", "self.TOOL_CALL_LENIENT_PATTERN.findall", "self.TOOL_CALL_LENIENT_PATTERN", "self.TOOL_CALL_LENIENT_PATTERN.sub", "self.RAW_JSON_TOOL_PATTERN.findall", "self.RAW_JSON_TOOL_PATTERN", "self.RAW_JSON_TOOL_PATTERN.sub" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=cleaned_text if cleaned_text else None)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=cleaned_text)" ], "awaits": false, "yields": false }, { "name": "_format_streaming_tool_calls", "qualname": "HermesToolParser._format_streaming_tool_calls", "full_name": "vllm_mlx.tool_parsers.hermes_tool_parser.HermesToolParser._format_streaming_tool_calls", "kind": "method", "signature": "def _format_streaming_tool_calls(tool_calls: list[dict], start_index: int=0) -> dict[str, Any]", "parameters": [ { "name": "tool_calls", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "start_index", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "dict[str, Any]", "docstring": "Format tool calls for streaming response.", "summary": "Format tool calls for streaming response.", "implementation": "Method `HermesToolParser._format_streaming_tool_calls` calls `enumerate`; returns `{'tool_calls': [{'index': start_index + i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'argume…`.", "documented": true, "public": false, "addressable": true, "line": 248, "end_line": 265, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/hermes_tool_parser.py#L248-L265", "decorators": [ "staticmethod" ], "calls": [ "enumerate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "{'tool_calls': [{'index': start_index + i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'argume…" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "HermesToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.hermes_tool_parser.HermesToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Hermes model output.\n\nUses tag counting to correctly handle multiple sequential tool calls.", "summary": "Extract tool calls from streaming Hermes model output.", "implementation": "Method `HermesToolParser.extract_tool_calls_streaming` calls `current_text.count`, `previous_text.count`, `self.extract_tool_calls`, `self._format_streaming_tool_calls`; has 5 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 267, "end_line": 336, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/hermes_tool_parser.py#L267-L336", "decorators": [], "calls": [ "current_text.count", "previous_text.count", "self.extract_tool_calls", "self._format_streaming_tool_calls", "delta_text.rstrip().endswith", "delta_text.rstrip" ], "state_reads": [ "self.extract_tool_calls", "self._format_streaming_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "self._format_streaming_tool_calls(new_calls, start_index=prev_close_count)", "{'content': delta_text}", "self._format_streaming_tool_calls(new_calls, start_index=prev_func_close)", "self._format_streaming_tool_calls(result.tool_calls)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.kimi_tool_parser", "path": "vllm_mlx/tool_parsers/kimi_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/kimi_tool_parser.md", "docstring": "Kimi/Moonshot tool call parser for vllm-mlx.\n\nHandles Kimi K2 and related models' tool calling format:\n- <|tool_calls_section_begin|>...<|tool_calls_section_end|>\n- <|tool_call_begin|>func_name:0<|tool_call_argument_begin|>{...}<|tool_call_end|>", "summary": "Kimi/Moonshot tool call parser for vllm-mlx.", "line_count": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/kimi_tool_parser.py#L1-L160", "members": [ "generate_tool_id", "KimiToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.kimi_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 23, "end_line": 25, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/kimi_tool_parser.py#L23-L25", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "KimiToolParser", "qualname": "KimiToolParser", "full_name": "vllm_mlx.tool_parsers.kimi_tool_parser.KimiToolParser", "kind": "class", "signature": "class KimiToolParser(ToolParser)", "parameters": [], "return_annotation": "KimiToolParser", "docstring": "Tool call parser for Kimi K2 and Moonshot models.\n\nSupports Kimi's tool call format:\n<|tool_calls_section_begin|>\n<|tool_call_begin|>func:0<|tool_call_argument_begin|>{...}<|tool_call_end|>\n<|tool_calls_section_end|>\n\nUsed when --enable-auto-tool-choice --tool-call-parser kimi are set.", "summary": "Tool call parser for Kimi K2 and Moonshot models.", "implementation": "Class `KimiToolParser` derives from `ToolParser` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 29, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/kimi_tool_parser.py#L29-L160", "decorators": [ "ToolParserManager.register_module(['kimi', 'kimi_k2', 'moonshot'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_has_tool_section", "qualname": "KimiToolParser._has_tool_section", "full_name": "vllm_mlx.tool_parsers.kimi_tool_parser.KimiToolParser._has_tool_section", "kind": "method", "signature": "def _has_tool_section(self, text: str) -> bool", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if text contains tool section markers.", "summary": "Check if text contains tool section markers.", "implementation": "Method `KimiToolParser._has_tool_section` returns `self.TOOL_CALLS_START in text or self.TOOL_CALLS_START_ALT in text or self.TOOL_CALL_START in text`.", "documented": true, "public": false, "addressable": true, "line": 59, "end_line": 65, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/kimi_tool_parser.py#L59-L65", "decorators": [], "calls": [], "state_reads": [ "self.TOOL_CALLS_START", "self.TOOL_CALLS_START_ALT", "self.TOOL_CALL_START" ], "state_writes": [], "raises": [], "return_expressions": [ "self.TOOL_CALLS_START in text or self.TOOL_CALLS_START_ALT in text or self.TOOL_CALL_START in text" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "KimiToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.kimi_tool_parser.KimiToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from Kimi model output.", "summary": "Extract tool calls from Kimi model output.", "implementation": "Method `KimiToolParser.extract_tool_calls` calls `self._has_tool_section`, `ExtractedToolCallInformation`, `model_output.find`, `model_output[:idx].strip`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 67, "end_line": 124, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/kimi_tool_parser.py#L67-L124", "decorators": [], "calls": [ "self._has_tool_section", "ExtractedToolCallInformation", "model_output.find", "model_output[:idx].strip", "self.TOOL_CALL_PATTERN.findall", "func_id.split", "func_name.split", "json.loads", "tool_calls.append", "generate_tool_id", "func_name.strip", "func_args.strip" ], "state_reads": [ "self._has_tool_section", "self.TOOL_CALLS_START", "self.TOOL_CALLS_START_ALT", "self.TOOL_CALL_PATTERN.findall", "self.TOOL_CALL_PATTERN" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "KimiToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.kimi_tool_parser.KimiToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Kimi model output.", "summary": "Extract tool calls from streaming Kimi model output.", "implementation": "Method `KimiToolParser.extract_tool_calls_streaming` calls `self._has_tool_section`, `self.extract_tool_calls`, `enumerate`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 126, "end_line": 160, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/kimi_tool_parser.py#L126-L160", "decorators": [], "calls": [ "self._has_tool_section", "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self._has_tool_section", "self.TOOL_CALL_END", "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.llama_tool_parser", "path": "vllm_mlx/tool_parsers/llama_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/llama_tool_parser.md", "docstring": "Llama tool call parser for vllm-mlx.\n\nHandles Llama's tool calling format:\n- XML style: {\"arg\": \"value\"}", "summary": "Llama tool call parser for vllm-mlx.", "line_count": 128, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/llama_tool_parser.py#L1-L128", "members": [ "generate_tool_id", "LlamaToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.llama_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 22, "end_line": 24, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/llama_tool_parser.py#L22-L24", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "LlamaToolParser", "qualname": "LlamaToolParser", "full_name": "vllm_mlx.tool_parsers.llama_tool_parser.LlamaToolParser", "kind": "class", "signature": "class LlamaToolParser(ToolParser)", "parameters": [], "return_annotation": "LlamaToolParser", "docstring": "Tool call parser for Llama models.\n\nSupports Llama tool call format:\n- {\"arg\": \"value\"}\n\nUsed when --enable-auto-tool-choice --tool-call-parser llama are set.", "summary": "Tool call parser for Llama models.", "implementation": "Class `LlamaToolParser` derives from `ToolParser` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 28, "end_line": 128, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/llama_tool_parser.py#L28-L128", "decorators": [ "ToolParserManager.register_module(['llama', 'llama3', 'llama4'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "LlamaToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.llama_tool_parser.LlamaToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete Llama model response.", "summary": "Extract tool calls from a complete Llama model response.", "implementation": "Method `LlamaToolParser.extract_tool_calls` calls `self.FUNCTION_PATTERN.findall`, `json.loads`, `tool_calls.append`, `generate_tool_id`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 44, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/llama_tool_parser.py#L44-L90", "decorators": [], "calls": [ "self.FUNCTION_PATTERN.findall", "json.loads", "tool_calls.append", "generate_tool_id", "name.strip", "isinstance", "json.dumps", "str", "self.FUNCTION_PATTERN.sub('', cleaned_text).strip", "self.FUNCTION_PATTERN.sub", "ExtractedToolCallInformation" ], "state_reads": [ "self.FUNCTION_PATTERN.findall", "self.FUNCTION_PATTERN", "self.FUNCTION_PATTERN.sub" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=cleaned_text if cleaned_text else None)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "LlamaToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.llama_tool_parser.LlamaToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Llama model output.", "summary": "Extract tool calls from streaming Llama model output.", "implementation": "Method `LlamaToolParser.extract_tool_calls_streaming` calls `self.extract_tool_calls`, `enumerate`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 92, "end_line": 128, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/llama_tool_parser.py#L92-L128", "decorators": [], "calls": [ "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.minimax_tool_parser", "path": "vllm_mlx/tool_parsers/minimax_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/minimax_tool_parser.md", "docstring": "MiniMax tool call parser for vllm-mlx.\n\nParses the MiniMax-M2 native XML tool call format:\n\n\nparam-value\n\n", "summary": "MiniMax tool call parser for vllm-mlx.", "line_count": 178, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/minimax_tool_parser.py#L1-L178", "members": [ "generate_tool_id", "MiniMaxToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.minimax_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Return a short OpenAI-compatible identifier for a parsed tool call.", "summary": "Return a short OpenAI-compatible identifier for a parsed tool call.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 26, "end_line": 29, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/minimax_tool_parser.py#L26-L29", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "MiniMaxToolParser", "qualname": "MiniMaxToolParser", "full_name": "vllm_mlx.tool_parsers.minimax_tool_parser.MiniMaxToolParser", "kind": "class", "signature": "class MiniMaxToolParser(ToolParser)", "parameters": [], "return_annotation": "MiniMaxToolParser", "docstring": "Parser for MiniMax-M2 tool call format.\n\nFormat:\n \n \n value\n \n ", "summary": "Parser for MiniMax-M2 tool call format.", "implementation": "Class `MiniMaxToolParser` derives from `ToolParser` and declares 5 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 33, "end_line": 178, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/minimax_tool_parser.py#L33-L178", "decorators": [ "ToolParserManager.register_module(['minimax', 'minimax_m2'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_extract_invokes", "qualname": "MiniMaxToolParser._extract_invokes", "full_name": "vllm_mlx.tool_parsers.minimax_tool_parser.MiniMaxToolParser._extract_invokes", "kind": "method", "signature": "def _extract_invokes(self, text: str) -> list[dict[str, Any]]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict[str, Any]]", "docstring": "Extract tool calls from invoke elements, with or without wrapper.", "summary": "Extract tool calls from invoke elements, with or without wrapper.", "implementation": "Method `MiniMaxToolParser._extract_invokes` calls `self.INVOKE_PATTERN.findall`, `self.PARAM_PATTERN.findall`, `p_value.strip`, `json.loads`; returns `tool_calls`.", "documented": true, "public": false, "addressable": true, "line": 54, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/minimax_tool_parser.py#L54-L78", "decorators": [], "calls": [ "self.INVOKE_PATTERN.findall", "self.PARAM_PATTERN.findall", "p_value.strip", "json.loads", "tool_calls.append", "generate_tool_id", "func_name.strip", "json.dumps" ], "state_reads": [ "self.INVOKE_PATTERN.findall", "self.INVOKE_PATTERN", "self.PARAM_PATTERN.findall", "self.PARAM_PATTERN" ], "state_writes": [], "raises": [], "return_expressions": [ "tool_calls" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "MiniMaxToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.minimax_tool_parser.MiniMaxToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract wrapped or bare MiniMax invoke elements from complete output.", "summary": "Extract wrapped or bare MiniMax invoke elements from complete output.", "implementation": "Method `MiniMaxToolParser.extract_tool_calls` calls `self.TOOL_CALL_BLOCK.findall`, `tool_calls.extend`, `self._extract_invokes`, `self.TOOL_CALL_BLOCK.sub('', model_output).strip`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 80, "end_line": 121, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/minimax_tool_parser.py#L80-L121", "decorators": [], "calls": [ "self.TOOL_CALL_BLOCK.findall", "tool_calls.extend", "self._extract_invokes", "self.TOOL_CALL_BLOCK.sub('', model_output).strip", "self.TOOL_CALL_BLOCK.sub", "self.THINK_PATTERN.sub('', cleaned).strip", "self.THINK_PATTERN.sub", "re.sub('\\\\[e~\\\\[.*$', '', cleaned).strip", "re.sub", "ExtractedToolCallInformation", "bool", "self.INVOKE_PATTERN.sub('', model_output).strip", "self.INVOKE_PATTERN.sub", "cleaned.replace('', '').strip", "cleaned.replace" ], "state_reads": [ "self.TOOL_CALL_BLOCK.findall", "self.TOOL_CALL_BLOCK", "self._extract_invokes", "self.TOOL_CALL_BLOCK.sub", "self.THINK_PATTERN.sub", "self.THINK_PATTERN", "self.INVOKE_PATTERN.sub", "self.INVOKE_PATTERN" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=bool(tool_calls), tool_calls=tool_calls, content=cleaned if cleaned else None)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=cleaned if cleaned else None)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)" ], "awaits": false, "yields": false }, { "name": "_has_tool_start", "qualname": "MiniMaxToolParser._has_tool_start", "full_name": "vllm_mlx.tool_parsers.minimax_tool_parser.MiniMaxToolParser._has_tool_start", "kind": "method", "signature": "def _has_tool_start(self, text: str) -> bool", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if text contains the start of a tool call block.", "summary": "Check if text contains the start of a tool call block.", "implementation": "Method `MiniMaxToolParser._has_tool_start` calls `self.INVOKE_PATTERN.search`; returns `'' in text or ('' in text or (' bool", "parameters": [ { "name": "current", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if a tool call block just completed.", "summary": "Check if a tool call block just completed.", "implementation": "Method `MiniMaxToolParser._has_tool_end` has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 129, "end_line": 140, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/minimax_tool_parser.py#L129-L140", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'' in current and '' not in previous", "True", "False" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "MiniMaxToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.minimax_tool_parser.MiniMaxToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Emit content deltas or a completed MiniMax tool-call delta.", "summary": "Emit content deltas or a completed MiniMax tool-call delta.", "implementation": "Method `MiniMaxToolParser.extract_tool_calls_streaming` calls `self._has_tool_start`, `self._has_tool_end`, `self.extract_tool_calls`, `enumerate`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 142, "end_line": 178, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/minimax_tool_parser.py#L142-L178", "decorators": [], "calls": [ "self._has_tool_start", "self._has_tool_end", "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self._has_tool_start", "self._has_tool_end", "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.mistral_tool_parser", "path": "vllm_mlx/tool_parsers/mistral_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/mistral_tool_parser.md", "docstring": "Mistral tool call parser for vllm-mlx.\n\nHandles Mistral's tool calling format:\n- Format: ``[TOOL_CALLS] [{\"name\": \"func\", \"arguments\": {...}}]``\n- Or newer: ``[TOOL_CALLS]func_name{\"arg\": \"value\"}``\n- Or newest (Ministral 3, Devstral Small 2, Dec 2025 tokenizers):\n ``[TOOL_CALLS]func_name[ARGS]{\"arg\": \"value\"}``\n Confirmed directly in these models' chat_template.jinja:\n ``{{- '[TOOL_CALLS]' + tool['function']['name'] + '[ARGS]' + arguments }}``\n\nUsed with models like Mistral-7B-Instruct, Devstral, Ministral 3, etc.", "summary": "Mistral tool call parser for vllm-mlx.", "line_count": 512, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L1-L512", "members": [ "ALPHANUMERIC", "_TOOL_NAME_PATTERN", "generate_mistral_tool_id", "_is_plain_tool_name", "MistralToolParser" ], "symbols": [ { "name": "generate_mistral_tool_id", "qualname": "generate_mistral_tool_id", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.generate_mistral_tool_id", "kind": "function", "signature": "def generate_mistral_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a random Mistral-compatible tool call ID.\n\nMistral Tool Call IDs must be alphanumeric with a length of 9.", "summary": "Generate a random Mistral-compatible tool call ID.", "implementation": "Function `generate_mistral_tool_id` calls `''.join`, `choices`; returns `''.join(choices(ALPHANUMERIC, k=9))`.", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 40, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L34-L40", "decorators": [], "calls": [ "''.join", "choices" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''.join(choices(ALPHANUMERIC, k=9))" ], "awaits": false, "yields": false }, { "name": "_is_plain_tool_name", "qualname": "_is_plain_tool_name", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser._is_plain_tool_name", "kind": "function", "signature": "def _is_plain_tool_name(name: str) -> bool", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True for names that are safe to dispatch as function calls.", "summary": "Return True for names that are safe to dispatch as function calls.", "implementation": "Function `_is_plain_tool_name` calls `bool`, `_TOOL_NAME_PATTERN.match`; returns `bool(_TOOL_NAME_PATTERN.match(name))`.", "documented": true, "public": false, "addressable": true, "line": 43, "end_line": 45, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L43-L45", "decorators": [], "calls": [ "bool", "_TOOL_NAME_PATTERN.match" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "bool(_TOOL_NAME_PATTERN.match(name))" ], "awaits": false, "yields": false }, { "name": "MistralToolParser", "qualname": "MistralToolParser", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser", "kind": "class", "signature": "class MistralToolParser(ToolParser)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "MistralToolParser", "docstring": "Tool call parser for Mistral models.\n\nSupports both old and new Mistral tool call formats:\n- Old (< v11): ``[TOOL_CALLS] [{\"name\": \"add\", \"arguments\": {\"a\": 1, \"b\": 2}}]``\n- New (>= v11): ``[TOOL_CALLS]add{\"a\": 1, \"b\": 2}``\n\nUsed when --enable-auto-tool-choice --tool-call-parser mistral are set.", "summary": "Tool call parser for Mistral models.", "implementation": "Class `MistralToolParser` derives from `ToolParser` and declares 8 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 49, "end_line": 512, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L49-L512", "decorators": [ "ToolParserManager.register_module('mistral')" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MistralToolParser.__init__", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `MistralToolParser.__init__` updates `self.bot_token_id`, `self._args_started`, `self._args_in_string`, `self._args_escaped`; calls `super().__init__`, `super`, `self.vocab.get`.", "implementation": "Method `MistralToolParser.__init__` updates `self.bot_token_id`, `self._args_started`, `self._args_in_string`, `self._args_escaped`; calls `super().__init__`, `super`, `self.vocab.get`.", "documented": false, "public": false, "addressable": true, "line": 68, "end_line": 90, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L68-L90", "decorators": [], "calls": [ "super().__init__", "super", "self.vocab.get" ], "state_reads": [ "self.vocab", "self.vocab.get", "self.BOT_TOKEN" ], "state_writes": [ "self.bot_token_id", "self._args_started", "self._args_in_string", "self._args_escaped", "self._name_buffer", "self._name_buffer_overflow", "self._current_tool_call_id", "self._tool_call_id_emitted" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset", "qualname": "MistralToolParser.reset", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser.reset", "kind": "method", "signature": "def reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset shared and Mistral-specific streaming tool-call state.", "summary": "Reset shared and Mistral-specific streaming tool-call state.", "implementation": "Method `MistralToolParser.reset` updates `self._args_started`, `self._args_in_string`, `self._args_escaped`, `self._name_buffer`; calls `super().reset`, `super`.", "documented": true, "public": true, "addressable": true, "line": 92, "end_line": 102, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L92-L102", "decorators": [], "calls": [ "super().reset", "super" ], "state_reads": [], "state_writes": [ "self._args_started", "self._args_in_string", "self._args_escaped", "self._name_buffer", "self._name_buffer_overflow", "self._current_tool_call_id", "self._tool_call_id_emitted" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_start_new_tool_call", "qualname": "MistralToolParser._start_new_tool_call", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser._start_new_tool_call", "kind": "method", "signature": "def _start_new_tool_call(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Begin a new streaming tool call: bump the index and reset the\nper-call name/arguments and id state.", "summary": "Begin a new streaming tool call: bump the index and reset the per-call name/arguments and id state.", "implementation": "Method `MistralToolParser._start_new_tool_call` updates `self.current_tool_id`, `self._args_started`, `self._args_in_string`, `self._args_escaped`; calls `generate_mistral_tool_id`.", "documented": true, "public": false, "addressable": true, "line": 104, "end_line": 114, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L104-L114", "decorators": [], "calls": [ "generate_mistral_tool_id" ], "state_reads": [], "state_writes": [ "self.current_tool_id", "self._args_started", "self._args_in_string", "self._args_escaped", "self._name_buffer", "self._name_buffer_overflow", "self._current_tool_call_id", "self._tool_call_id_emitted" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_scan_args_for_new_call", "qualname": "MistralToolParser._scan_args_for_new_call", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser._scan_args_for_new_call", "kind": "method", "signature": "def _scan_args_for_new_call(self, text: str) -> int", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Scan an argument delta, updating the persistent JSON string state,\nand return the position of the first [TOOL_CALLS] marker that sits\noutside a string (a new call), or -1 when there is none.\n\nQuote state is carried across deltas so a marker inside a quoted\nvalue (e.g. ``{\"city\": \"[TOOL_CALLS]rm\"}``) stays argument data while\na marker between two calls opens the next index.", "summary": "Scan an argument delta, updating the persistent JSON string state, and return the position of the first [TOOL_CALLS] marker that sits outside a string (a new call), or -1 when there is none.", "implementation": "Method `MistralToolParser._scan_args_for_new_call` updates `self._args_escaped`, `self._args_in_string`; calls `len`, `text.startswith`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 116, "end_line": 146, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L116-L146", "decorators": [], "calls": [ "len", "text.startswith" ], "state_reads": [ "self._args_escaped", "self._args_in_string", "self.BOT_TOKEN" ], "state_writes": [ "self._args_escaped", "self._args_in_string" ], "raises": [], "return_expressions": [ "i", "-1" ], "awaits": false, "yields": false }, { "name": "_split_on_tool_call_markers", "qualname": "MistralToolParser._split_on_tool_call_markers", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser._split_on_tool_call_markers", "kind": "method", "signature": "def _split_on_tool_call_markers(self, text: str) -> list[str]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[str]", "docstring": "Split on [TOOL_CALLS] occurrences that are outside JSON strings.\n\nA marker appearing inside a quoted string value is argument data,\nnot a new call — splitting there would let untrusted model output\nforge a second dispatchable call.\n\nThe quote-state scan starts at the first marker, not at index 0: the\ntext before the first marker is prose, not JSON, so an odd number of\ndouble quotes there must not leave ``in_string`` set when the marker\narrives (that would hide the call entirely).", "summary": "Split on [TOOL_CALLS] occurrences that are outside JSON strings.", "implementation": "Method `MistralToolParser._split_on_tool_call_markers` calls `text.find`, `len`, `text.startswith`, `parts.append`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 148, "end_line": 192, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L148-L192", "decorators": [], "calls": [ "text.find", "len", "text.startswith", "parts.append" ], "state_reads": [ "self.BOT_TOKEN" ], "state_writes": [], "raises": [], "return_expressions": [ "[text]", "parts" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "MistralToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "The complete model output string" }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional request context" } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete Mistral model response.\n\nArgs:\n model_output: The complete model output string\n request: Optional request context\n\nReturns:\n ExtractedToolCallInformation with parsed tool calls", "summary": "Extract tool calls from a complete Mistral model response.", "implementation": "Method `MistralToolParser.extract_tool_calls` calls `ExtractedToolCallInformation`, `self._split_on_tool_call_markers`, `content_and_raw_tool_calls[0].strip`, `raw_tool_call.strip`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 194, "end_line": 332, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L194-L332", "decorators": [], "calls": [ "ExtractedToolCallInformation", "self._split_on_tool_call_markers", "content_and_raw_tool_calls[0].strip", "raw_tool_call.strip", "raw_tool_call.find", "raw_tool_call.startswith", "raw_tool_call[:args_idx].strip", "len", "_is_plain_tool_name", "json.loads", "tool_calls.append", "generate_mistral_tool_id", "raw_tool_call[:end_name].strip", "isinstance", "item.get", "json.dumps", "str", "self.TOOL_CALL_REGEX.search", "match.group", "(content + ' ' + raw_tool_call).strip" ], "state_reads": [ "self.BOT_TOKEN", "self._split_on_tool_call_markers", "self.ARGS_TOKEN", "self.TOOL_CALL_REGEX.search", "self.TOOL_CALL_REGEX" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content if content else None)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "MistralToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Mistral model output.\n\nFor streaming, we detect when [TOOL_CALLS] appears and start\naccumulating tool call data.", "summary": "Extract tool calls from streaming Mistral model output.", "implementation": "Method `MistralToolParser.extract_tool_calls_streaming` updates `self._tool_call_id_emitted`; calls `self._scan_args_for_new_call`, `self._start_new_tool_call`, `self._parse_streaming_tool_delta`, `len`; has 6 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 334, "end_line": 453, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L334-L453", "decorators": [], "calls": [ "self._scan_args_for_new_call", "self._start_new_tool_call", "self._parse_streaming_tool_delta", "len", "result.get", "delta_text.split", "self.BOT_TOKEN.join" ], "state_reads": [ "self._args_started", "self._scan_args_for_new_call", "self.current_tool_id", "self._start_new_tool_call", "self._parse_streaming_tool_delta", "self.BOT_TOKEN", "self._current_tool_call_id", "self.BOT_TOKEN.join", "self._name_buffer_overflow", "self._tool_call_id_emitted" ], "state_writes": [ "self._tool_call_id_emitted" ], "raises": [], "return_expressions": [ "{'tool_calls': [{'index': self.current_tool_id, 'type': 'function', 'function': {'arguments': delta_text}}]}", "result if result else None", "{'content': delta_text}", "{'content': tool_delta['content']}", "{'tool_calls': [tool_call]}", "None" ], "awaits": false, "yields": false }, { "name": "_parse_streaming_tool_delta", "qualname": "MistralToolParser._parse_streaming_tool_delta", "full_name": "vllm_mlx.tool_parsers.mistral_tool_parser.MistralToolParser._parse_streaming_tool_delta", "kind": "method", "signature": "def _parse_streaming_tool_delta(self, text: str) -> dict[str, str] | None", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "dict[str, str] | None", "docstring": "Parse a streaming delta for tool call information.\n\nOnce the name/arguments boundary (the `[ARGS]` marker, or a bare `{`\nfor older checkpoints) has been seen for the current tool call, every\nsubsequent delta is argument text and is never re-classified — JSON\nstring content (bare keys/values like `city` or `Paris`) has no\ndistinguishing leading punctuation, so re-evaluating each delta in\nisolation (the previous approach) misclassified mid-argument\nfragments as more of the function name.", "summary": "Parse a streaming delta for tool call information.", "implementation": "Method `MistralToolParser._parse_streaming_tool_delta` updates `self._name_buffer`, `self._args_started`, `self._name_buffer_overflow`; calls `self._name_buffer.find`, `len`, `self._name_buffer[:idx].strip`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 455, "end_line": 512, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/mistral_tool_parser.py#L455-L512", "decorators": [], "calls": [ "self._name_buffer.find", "len", "self._name_buffer[:idx].strip" ], "state_reads": [ "self._args_started", "self._name_buffer.find", "self._name_buffer", "self.ARGS_TOKEN", "self._NAME_BUFFER_LIMIT" ], "state_writes": [ "self._name_buffer", "self._args_started", "self._name_buffer_overflow" ], "raises": [], "return_expressions": [ "None", "{'arguments': text}", "result if result else None", "{'content': overflowed}" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.nemotron_tool_parser", "path": "vllm_mlx/tool_parsers/nemotron_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/nemotron_tool_parser.md", "docstring": "Nemotron tool call parser for vllm-mlx.\n\nHandles NVIDIA Nemotron models' tool calling format:\n- v\n\nSupports Nemotron-3-Nano-30B-A3B and similar models.", "summary": "Nemotron tool call parser for vllm-mlx.", "line_count": 166, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/nemotron_tool_parser.py#L1-L166", "members": [ "generate_tool_id", "NemotronToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.nemotron_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 24, "end_line": 26, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/nemotron_tool_parser.py#L24-L26", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "NemotronToolParser", "qualname": "NemotronToolParser", "full_name": "vllm_mlx.tool_parsers.nemotron_tool_parser.NemotronToolParser", "kind": "class", "signature": "class NemotronToolParser(ToolParser)", "parameters": [], "return_annotation": "NemotronToolParser", "docstring": "Tool call parser for NVIDIA Nemotron models.\n\nSupports Nemotron's tool call format:\nParis\n\nAlso supports JSON arguments:\n{\"city\": \"Paris\"}\n\nUsed when --enable-auto-tool-choice --tool-call-parser nemotron are set.", "summary": "Tool call parser for NVIDIA Nemotron models.", "implementation": "Class `NemotronToolParser` derives from `ToolParser` and declares 2 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 30, "end_line": 166, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/nemotron_tool_parser.py#L30-L166", "decorators": [ "ToolParserManager.register_module(['nemotron', 'nemotron3'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "NemotronToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.nemotron_tool_parser.NemotronToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from Nemotron model output.", "summary": "Extract tool calls from Nemotron model output.", "implementation": "Method `NemotronToolParser.extract_tool_calls` calls `ExtractedToolCallInformation`, `self.TOOL_CALL_PATTERN.findall`, `func_name.strip`, `content.strip`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 55, "end_line": 130, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/nemotron_tool_parser.py#L55-L130", "decorators": [], "calls": [ "ExtractedToolCallInformation", "self.TOOL_CALL_PATTERN.findall", "func_name.strip", "content.strip", "content.startswith", "json.loads", "tool_calls.append", "generate_tool_id", "self.PARAM_PATTERN.findall", "param_name.strip", "param_value.strip", "json.dumps", "self.TOOL_CALL_PATTERN.sub('', cleaned_text).strip", "self.TOOL_CALL_PATTERN.sub" ], "state_reads": [ "self.TOOL_CALL_PATTERN.findall", "self.TOOL_CALL_PATTERN", "self.PARAM_PATTERN.findall", "self.PARAM_PATTERN", "self.TOOL_CALL_PATTERN.sub" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=cleaned_text if cleaned_text else None)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "NemotronToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.nemotron_tool_parser.NemotronToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Nemotron model output.", "summary": "Extract tool calls from streaming Nemotron model output.", "implementation": "Method `NemotronToolParser.extract_tool_calls_streaming` calls `self.extract_tool_calls`, `enumerate`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 132, "end_line": 166, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/nemotron_tool_parser.py#L132-L166", "decorators": [], "calls": [ "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser", "path": "vllm_mlx/tool_parsers/poolside_v1_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/poolside_v1_tool_parser.md", "docstring": "Tool parser for the Poolside v1 Laguna chat-template format.", "summary": "Tool parser for the Poolside v1 Laguna chat-template format.", "line_count": 362, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L1-L362", "members": [ "_consume_stream_state", "PoolsideV1ToolParser" ], "symbols": [ { "name": "_consume_stream_state", "qualname": "_consume_stream_state", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser._consume_stream_state", "kind": "function", "signature": "def _consume_stream_state(parser, pending: dict[int, dict[str, Any]], valid_names: set[str], request: dict[str, Any] | None) -> tuple[bool, str]", "parameters": [ { "name": "parser", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "pending", "kind": "positional or keyword", "annotation": "dict[int, dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "valid_names", "kind": "positional or keyword", "annotation": "set[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[bool, str]", "docstring": "", "summary": "Function `_consume_stream_state` calls `parser._consume_text_before_tool`, `parser._consume_tool_name`, `parser._consume_string_value`, `parser._consume_pending_key`; has 5 explicit return paths.", "implementation": "Function `_consume_stream_state` calls `parser._consume_text_before_tool`, `parser._consume_tool_name`, `parser._consume_string_value`, `parser._consume_pending_key`; has 5 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 16, "end_line": 30, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L16-L30", "decorators": [], "calls": [ "parser._consume_text_before_tool", "parser._consume_tool_name", "parser._consume_string_value", "parser._consume_pending_key", "parser._consume_tool_body" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parser._consume_text_before_tool()", "(parser._consume_tool_name(pending, valid_names), '')", "(parser._consume_string_value(pending), '')", "(parser._consume_pending_key(pending, request), '')", "(parser._consume_tool_body(pending), '')" ], "awaits": false, "yields": false }, { "name": "PoolsideV1ToolParser", "qualname": "PoolsideV1ToolParser", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser", "kind": "class", "signature": "class PoolsideV1ToolParser(Glm47ToolParser)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "PoolsideV1ToolParser", "docstring": "Parse Laguna tool calls and stream schema-declared strings incrementally.", "summary": "Parse Laguna tool calls and stream schema-declared strings incrementally.", "implementation": "Class `PoolsideV1ToolParser` derives from `Glm47ToolParser` and declares 18 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 362, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L34-L362", "decorators": [ "ToolParserManager.register_module('poolside_v1')" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "PoolsideV1ToolParser.__init__", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `PoolsideV1ToolParser.__init__` calls `super().__init__`, `super`, `self.reset`.", "implementation": "Method `PoolsideV1ToolParser.__init__` calls `super().__init__`, `super`, `self.reset`.", "documented": false, "public": false, "addressable": true, "line": 45, "end_line": 47, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L45-L47", "decorators": [], "calls": [ "super().__init__", "super", "self.reset" ], "state_reads": [ "self.reset" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset", "qualname": "PoolsideV1ToolParser.reset", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser.reset", "kind": "method", "signature": "def reset(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Reset Laguna parser buffers and per-call argument state.", "summary": "Reset Laguna parser buffers and per-call argument state.", "implementation": "Method `PoolsideV1ToolParser.reset` updates `self._buffer`, `self._in_tool_call`, `self._current_tool_name`, `self._pending_key`; calls `super().reset`, `super`.", "documented": true, "public": true, "addressable": true, "line": 49, "end_line": 62, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L49-L62", "decorators": [], "calls": [ "super().reset", "super" ], "state_reads": [], "state_writes": [ "self._buffer", "self._in_tool_call", "self._current_tool_name", "self._pending_key", "self._streaming_string_value", "self._reject_current", "self._tool_ids", "self._args_started", "self._args_closed", "self._seen_keys" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_string_argument_names", "qualname": "PoolsideV1ToolParser._string_argument_names", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._string_argument_names", "kind": "method", "signature": "def _string_argument_names(request: dict[str, Any] | None, tool_name: str) -> set[str]", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tool_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "set[str]", "docstring": "", "summary": "Method `PoolsideV1ToolParser._string_argument_names` calls `set`, `request.get`, `isinstance`, `tool.get`; has 2 explicit return paths.", "implementation": "Method `PoolsideV1ToolParser._string_argument_names` calls `set`, `request.get`, `isinstance`, `tool.get`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 65, "end_line": 85, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L65-L85", "decorators": [ "staticmethod" ], "calls": [ "set", "request.get", "isinstance", "tool.get", "function.get", "parameters.get", "properties.items", "schema.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "set()", "{name for name, schema in properties.items() if isinstance(schema, dict) and schema.get('type') == 'string'}" ], "awaits": false, "yields": false }, { "name": "_escape_string_content", "qualname": "PoolsideV1ToolParser._escape_string_content", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._escape_string_content", "kind": "method", "signature": "def _escape_string_content(value: str) -> str", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "", "summary": "Method `PoolsideV1ToolParser._escape_string_content` calls `json.dumps`; returns `json.dumps(value, ensure_ascii=False)[1:-1]`.", "implementation": "Method `PoolsideV1ToolParser._escape_string_content` calls `json.dumps`; returns `json.dumps(value, ensure_ascii=False)[1:-1]`.", "documented": false, "public": false, "addressable": true, "line": 88, "end_line": 89, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L88-L89", "decorators": [ "staticmethod" ], "calls": [ "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.dumps(value, ensure_ascii=False)[1:-1]" ], "awaits": false, "yields": false }, { "name": "_hold_partial_suffix", "qualname": "PoolsideV1ToolParser._hold_partial_suffix", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._hold_partial_suffix", "kind": "method", "signature": "def _hold_partial_suffix(buffer: str, marker: str) -> tuple[str, str]", "parameters": [ { "name": "buffer", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "marker", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str, str]", "docstring": "", "summary": "Method `PoolsideV1ToolParser._hold_partial_suffix` calls `range`, `min`, `len`, `buffer.endswith`; has 2 explicit return paths.", "implementation": "Method `PoolsideV1ToolParser._hold_partial_suffix` calls `range`, `min`, `len`, `buffer.endswith`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 92, "end_line": 96, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L92-L96", "decorators": [ "staticmethod" ], "calls": [ "range", "min", "len", "buffer.endswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(buffer[:-size], buffer[-size:])", "(buffer, '')" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "PoolsideV1ToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract complete Laguna tool blocks and preserve remaining content.", "summary": "Extract complete Laguna tool blocks and preserve remaining content.", "implementation": "Method `PoolsideV1ToolParser.extract_tool_calls` calls `self.strip_think_tags`, `self._get_tool_names`, `self.FUNC_DETAIL_PATTERN.finditer`, `match.group(1).strip`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 98, "end_line": 138, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L98-L138", "decorators": [], "calls": [ "self.strip_think_tags", "self._get_tool_names", "self.FUNC_DETAIL_PATTERN.finditer", "match.group(1).strip", "match.group", "self._string_argument_names", "self.ARG_PATTERN.findall", "raw_key.strip", "self._deserialize", "raw_value.strip", "tool_calls.append", "generate_tool_id", "json.dumps", "cleaned_text.find", "content.strip", "ExtractedToolCallInformation", "self._UNCLOSED_TOOL_CALL.sub('', cleaned_text).strip", "self._UNCLOSED_TOOL_CALL.sub" ], "state_reads": [ "self.strip_think_tags", "self._get_tool_names", "self.FUNC_DETAIL_PATTERN.finditer", "self.FUNC_DETAIL_PATTERN", "self._string_argument_names", "self.ARG_PATTERN.findall", "self.ARG_PATTERN", "self._deserialize", "self._START", "self._UNCLOSED_TOOL_CALL.sub", "self._UNCLOSED_TOOL_CALL" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(True, tool_calls, content)", "ExtractedToolCallInformation(False, [], content)" ], "awaits": false, "yields": false }, { "name": "_begin_tool_call", "qualname": "PoolsideV1ToolParser._begin_tool_call", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._begin_tool_call", "kind": "method", "signature": "def _begin_tool_call(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `PoolsideV1ToolParser._begin_tool_call` updates `self.current_tool_id`, `self._in_tool_call`, `self._current_tool_name`, `self._pending_key`; calls `self._tool_ids.append`, `generate_tool_id`, `self._args_started.append`, `self._args_closed.append`.", "implementation": "Method `PoolsideV1ToolParser._begin_tool_call` updates `self.current_tool_id`, `self._in_tool_call`, `self._current_tool_name`, `self._pending_key`; calls `self._tool_ids.append`, `generate_tool_id`, `self._args_started.append`, `self._args_closed.append`.", "documented": false, "public": false, "addressable": true, "line": 140, "end_line": 150, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L140-L150", "decorators": [], "calls": [ "self._tool_ids.append", "generate_tool_id", "self._args_started.append", "self._args_closed.append", "self._seen_keys.append", "set" ], "state_reads": [ "self._tool_ids.append", "self._tool_ids", "self._args_started.append", "self._args_started", "self._args_closed.append", "self._args_closed", "self._seen_keys.append", "self._seen_keys" ], "state_writes": [ "self.current_tool_id", "self._in_tool_call", "self._current_tool_name", "self._pending_key", "self._streaming_string_value", "self._reject_current" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_finish_tool_call", "qualname": "PoolsideV1ToolParser._finish_tool_call", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._finish_tool_call", "kind": "method", "signature": "def _finish_tool_call(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "", "summary": "Method `PoolsideV1ToolParser._finish_tool_call` updates `self._in_tool_call`, `self._current_tool_name`, `self._pending_key`, `self._streaming_string_value`.", "implementation": "Method `PoolsideV1ToolParser._finish_tool_call` updates `self._in_tool_call`, `self._current_tool_name`, `self._pending_key`, `self._streaming_string_value`.", "documented": false, "public": false, "addressable": true, "line": 152, "end_line": 157, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L152-L157", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._in_tool_call", "self._current_tool_name", "self._pending_key", "self._streaming_string_value", "self._reject_current" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_delta", "qualname": "PoolsideV1ToolParser._delta", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._delta", "kind": "method", "signature": "def _delta(self, pending: dict[int, dict[str, Any]], *, name: str | None=None, arguments: str='') -> None", "parameters": [ { "name": "pending", "kind": "positional or keyword", "annotation": "dict[int, dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "name", "kind": "keyword-only", "annotation": "str | None", "default": "None", "required": false, "description": "Optional keyword-only input; defaults to `None`." }, { "name": "arguments", "kind": "keyword-only", "annotation": "str", "default": "''", "required": false, "description": "Optional keyword-only input; defaults to `''`." } ], "return_annotation": "None", "docstring": "", "summary": "Method `PoolsideV1ToolParser._delta` calls `pending.setdefault`; returns `None`.", "implementation": "Method `PoolsideV1ToolParser._delta` calls `pending.setdefault`; returns `None`.", "documented": false, "public": false, "addressable": true, "line": 159, "end_line": 179, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L159-L179", "decorators": [], "calls": [ "pending.setdefault" ], "state_reads": [ "self._reject_current", "self.current_tool_id", "self._tool_ids" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_argument_prefix", "qualname": "PoolsideV1ToolParser._argument_prefix", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._argument_prefix", "kind": "method", "signature": "def _argument_prefix(self, key: str) -> str | None", "parameters": [ { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str | None", "docstring": "", "summary": "Method `PoolsideV1ToolParser._argument_prefix` calls `seen.add`, `json.dumps`; has 2 explicit return paths.", "implementation": "Method `PoolsideV1ToolParser._argument_prefix` calls `seen.add`, `json.dumps`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 181, "end_line": 188, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L181-L188", "decorators": [], "calls": [ "seen.add", "json.dumps" ], "state_reads": [ "self._seen_keys", "self.current_tool_id", "self._args_started" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "separator + json.dumps(key, ensure_ascii=False) + ': '" ], "awaits": false, "yields": false }, { "name": "_close_arguments", "qualname": "PoolsideV1ToolParser._close_arguments", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._close_arguments", "kind": "method", "signature": "def _close_arguments(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `PoolsideV1ToolParser._close_arguments` has 2 explicit return paths.", "implementation": "Method `PoolsideV1ToolParser._close_arguments` has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 190, "end_line": 194, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L190-L194", "decorators": [], "calls": [], "state_reads": [ "self._args_closed", "self.current_tool_id", "self._args_started" ], "state_writes": [], "raises": [], "return_expressions": [ "''", "'}' if self._args_started[self.current_tool_id] else '{}'" ], "awaits": false, "yields": false }, { "name": "_discard_through_tool_end", "qualname": "PoolsideV1ToolParser._discard_through_tool_end", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._discard_through_tool_end", "kind": "method", "signature": "def _discard_through_tool_end(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "", "summary": "Method `PoolsideV1ToolParser._discard_through_tool_end` updates `self._buffer`; calls `self._buffer.find`, `len`, `self._finish_tool_call`; has 2 explicit return paths.", "implementation": "Method `PoolsideV1ToolParser._discard_through_tool_end` updates `self._buffer`; calls `self._buffer.find`, `len`, `self._finish_tool_call`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 196, "end_line": 202, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L196-L202", "decorators": [], "calls": [ "self._buffer.find", "len", "self._finish_tool_call" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._END", "self._finish_tool_call" ], "state_writes": [ "self._buffer" ], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_consume_text_before_tool", "qualname": "PoolsideV1ToolParser._consume_text_before_tool", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._consume_text_before_tool", "kind": "method", "signature": "def _consume_text_before_tool(self) -> tuple[bool, str]", "parameters": [], "return_annotation": "tuple[bool, str]", "docstring": "Consume plain text or enter the next ```` state.", "summary": "Consume plain text or enter the next ```` state.", "implementation": "Method `PoolsideV1ToolParser._consume_text_before_tool` updates `self._buffer`; calls `self._buffer.find`, `self._hold_partial_suffix`, `len`, `self._begin_tool_call`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 204, "end_line": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L204-L214", "decorators": [], "calls": [ "self._buffer.find", "self._hold_partial_suffix", "len", "self._begin_tool_call" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._START", "self._hold_partial_suffix", "self._begin_tool_call" ], "state_writes": [ "self._buffer" ], "raises": [], "return_expressions": [ "(False, emitted)", "(True, content)" ], "awaits": false, "yields": false }, { "name": "_consume_tool_name", "qualname": "PoolsideV1ToolParser._consume_tool_name", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._consume_tool_name", "kind": "method", "signature": "def _consume_tool_name(self, pending: dict[int, dict[str, Any]], valid_names: set[str]) -> bool", "parameters": [ { "name": "pending", "kind": "positional or keyword", "annotation": "dict[int, dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "valid_names", "kind": "positional or keyword", "annotation": "set[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Consume a tool name, or wait for enough input to identify it.", "summary": "Consume a tool name, or wait for enough input to identify it.", "implementation": "Method `PoolsideV1ToolParser._consume_tool_name` updates `self._buffer`, `self._reject_current`, `self._current_tool_name`; calls `self._buffer.find`, `min`, `self._buffer[:cut].strip`, `self._buffer.startswith`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 216, "end_line": 247, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L216-L247", "decorators": [], "calls": [ "self._buffer.find", "min", "self._buffer[:cut].strip", "self._buffer.startswith", "self._discard_through_tool_end", "self._delta" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._KEY_START", "self._END", "self._buffer.startswith", "self._discard_through_tool_end", "self._delta" ], "state_writes": [ "self._buffer", "self._reject_current", "self._current_tool_name" ], "raises": [], "return_expressions": [ "False", "self._discard_through_tool_end()", "True" ], "awaits": false, "yields": false }, { "name": "_consume_string_value", "qualname": "PoolsideV1ToolParser._consume_string_value", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._consume_string_value", "kind": "method", "signature": "def _consume_string_value(self, pending: dict[int, dict[str, Any]]) -> bool", "parameters": [ { "name": "pending", "kind": "positional or keyword", "annotation": "dict[int, dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Consume a string argument value, retaining incomplete suffixes.", "summary": "Consume a string argument value, retaining incomplete suffixes.", "implementation": "Method `PoolsideV1ToolParser._consume_string_value` updates `self._buffer`, `self._streaming_string_value`, `self._pending_key`, `self._reject_current`; calls `self._buffer.find`, `self._escape_string_content`, `len`, `self._delta`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 249, "end_line": 267, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L249-L267", "decorators": [], "calls": [ "self._buffer.find", "self._escape_string_content", "len", "self._delta", "self._discard_through_tool_end", "self._hold_partial_suffix" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._VALUE_END", "self._escape_string_content", "self._delta", "self._END", "self._discard_through_tool_end", "self._hold_partial_suffix" ], "state_writes": [ "self._buffer", "self._streaming_string_value", "self._pending_key", "self._reject_current" ], "raises": [], "return_expressions": [ "True", "self._discard_through_tool_end()", "False" ], "awaits": false, "yields": false }, { "name": "_consume_pending_key", "qualname": "PoolsideV1ToolParser._consume_pending_key", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._consume_pending_key", "kind": "method", "signature": "def _consume_pending_key(self, pending: dict[int, dict[str, Any]], request: dict[str, Any] | None) -> bool", "parameters": [ { "name": "pending", "kind": "positional or keyword", "annotation": "dict[int, dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Consume the value for the currently buffered argument key.", "summary": "Consume the value for the currently buffered argument key.", "implementation": "Method `PoolsideV1ToolParser._consume_pending_key` updates `self._reject_current`, `self._buffer`, `self._pending_key`, `self._streaming_string_value`; calls `self._buffer.find`, `self._discard_through_tool_end`, `len`, `self._pending_key.strip`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 269, "end_line": 305, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L269-L305", "decorators": [], "calls": [ "self._buffer.find", "self._discard_through_tool_end", "len", "self._pending_key.strip", "self._argument_prefix", "self._string_argument_names", "self._delta", "self._buffer[:value_end].strip", "json.dumps", "self._deserialize" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._VALUE_START", "self._END", "self._discard_through_tool_end", "self._pending_key", "self._pending_key.strip", "self._argument_prefix", "self._string_argument_names", "self._current_tool_name", "self._delta", "self._VALUE_END", "self._deserialize" ], "state_writes": [ "self._reject_current", "self._buffer", "self._pending_key", "self._streaming_string_value" ], "raises": [], "return_expressions": [ "self._discard_through_tool_end()", "False", "True" ], "awaits": false, "yields": false }, { "name": "_consume_tool_body", "qualname": "PoolsideV1ToolParser._consume_tool_body", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser._consume_tool_body", "kind": "method", "signature": "def _consume_tool_body(self, pending: dict[int, dict[str, Any]]) -> bool", "parameters": [ { "name": "pending", "kind": "positional or keyword", "annotation": "dict[int, dict[str, Any]]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Consume an argument key or close the current tool call.", "summary": "Consume an argument key or close the current tool call.", "implementation": "Method `PoolsideV1ToolParser._consume_tool_body` updates `self._buffer`, `self._pending_key`; calls `self._buffer.find`, `len`, `self._delta`, `self._close_arguments`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 307, "end_line": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L307-L328", "decorators": [], "calls": [ "self._buffer.find", "len", "self._delta", "self._close_arguments", "self._finish_tool_call" ], "state_reads": [ "self._buffer.find", "self._buffer", "self._END", "self._KEY_START", "self._delta", "self._close_arguments", "self._finish_tool_call", "self._KEY_END" ], "state_writes": [ "self._buffer", "self._pending_key" ], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "PoolsideV1ToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.poolside_v1_tool_parser.PoolsideV1ToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Incrementally emit Laguna content and schema-aware tool arguments.", "summary": "Incrementally emit Laguna content and schema-aware tool arguments.", "implementation": "Method `PoolsideV1ToolParser.extract_tool_calls_streaming` updates `self._buffer`; calls `self._get_tool_names`, `_consume_stream_state`, `list`, `pending.values`; returns `payload or None`.", "documented": true, "public": true, "addressable": true, "line": 330, "end_line": 362, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/poolside_v1_tool_parser.py#L330-L362", "decorators": [], "calls": [ "self._get_tool_names", "_consume_stream_state", "list", "pending.values" ], "state_reads": [ "self._get_tool_names" ], "state_writes": [ "self._buffer" ], "raises": [], "return_expressions": [ "payload or None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser", "path": "vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.md", "docstring": "Qwen 3.5 XML tool call parser for vllm-mlx.\n\nHandles Qwen 3.5's XML parameter format:\n\n \n \n value1\n \n \n\nAuthority: Qwen 3.5 HF chat template, vLLM PR #25028 (from Qwen API team).", "summary": "Qwen 3.5 XML tool call parser for vllm-mlx.", "line_count": 1559, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1-L1559", "members": [ "logger", "DeltaFunctionCall", "DeltaToolCall", "DeltaMessage", "ChatCompletionToolsParam", "_FunctionDef", "_ToolDef", "StreamingXMLToolCallParser", "Qwen3XMLToolParser" ], "symbols": [ { "name": "DeltaFunctionCall", "qualname": "DeltaFunctionCall", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.DeltaFunctionCall", "kind": "class", "signature": "class DeltaFunctionCall", "parameters": [ { "name": "name", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "arguments", "kind": "field", "annotation": "str", "default": "''", "required": false, "description": "Optional constructor field; defaults to `''`." } ], "return_annotation": "DeltaFunctionCall", "docstring": "Incremental function name and argument payload used by the XML parser.", "summary": "Incremental function name and argument payload used by the XML parser.", "implementation": "Class `DeltaFunctionCall` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 54, "end_line": 58, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L54-L58", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "DeltaToolCall", "qualname": "DeltaToolCall", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.DeltaToolCall", "kind": "class", "signature": "class DeltaToolCall", "parameters": [ { "name": "index", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "id", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "type", "kind": "field", "annotation": "str", "default": "'function'", "required": false, "description": "Optional constructor field; defaults to `'function'`." }, { "name": "function", "kind": "field", "annotation": "Optional[DeltaFunctionCall]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "DeltaToolCall", "docstring": "Incremental indexed tool call produced by the XML parser shim.", "summary": "Incremental indexed tool call produced by the XML parser shim.", "implementation": "Class `DeltaToolCall` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 62, "end_line": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L62-L68", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "DeltaMessage", "qualname": "DeltaMessage", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.DeltaMessage", "kind": "class", "signature": "class DeltaMessage", "parameters": [ { "name": "content", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "tool_calls", "kind": "field", "annotation": "Optional[list[DeltaToolCall]]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "role", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." }, { "name": "reasoning_content", "kind": "field", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional constructor field; defaults to `None`." } ], "return_annotation": "DeltaMessage", "docstring": "Incremental content, reasoning, and tool calls from the parser shim.", "summary": "Incremental content, reasoning, and tool calls from the parser shim.", "implementation": "Class `DeltaMessage` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 72, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L72-L78", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_FunctionDef", "qualname": "_FunctionDef", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._FunctionDef", "kind": "class", "signature": "class _FunctionDef", "parameters": [ { "name": "d", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_FunctionDef", "docstring": "Wrap a function definition dict for attribute access.", "summary": "Wrap a function definition dict for attribute access.", "implementation": "Class `_FunctionDef` declares 3 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 85, "end_line": 99, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L85-L99", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_FunctionDef.__init__", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._FunctionDef.__init__", "kind": "method", "signature": "def __init__(self, d: dict)", "parameters": [ { "name": "d", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_FunctionDef.__init__` updates `self._d`.", "implementation": "Method `_FunctionDef.__init__` updates `self._d`.", "documented": false, "public": false, "addressable": true, "line": 90, "end_line": 91, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L90-L91", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self._d" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "name", "qualname": "_FunctionDef.name", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._FunctionDef.name", "kind": "method", "signature": "def name(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `_FunctionDef.name` calls `self._d.get`; returns `self._d.get('name', '')`.", "implementation": "Method `_FunctionDef.name` calls `self._d.get`; returns `self._d.get('name', '')`.", "documented": false, "public": false, "addressable": true, "line": 94, "end_line": 95, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L94-L95", "decorators": [ "property" ], "calls": [ "self._d.get" ], "state_reads": [ "self._d.get", "self._d" ], "state_writes": [], "raises": [], "return_expressions": [ "self._d.get('name', '')" ], "awaits": false, "yields": false }, { "name": "parameters", "qualname": "_FunctionDef.parameters", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._FunctionDef.parameters", "kind": "method", "signature": "def parameters(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "", "summary": "Method `_FunctionDef.parameters` calls `self._d.get`; returns `self._d.get('parameters', {})`.", "implementation": "Method `_FunctionDef.parameters` calls `self._d.get`; returns `self._d.get('parameters', {})`.", "documented": false, "public": false, "addressable": true, "line": 98, "end_line": 99, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L98-L99", "decorators": [ "property" ], "calls": [ "self._d.get" ], "state_reads": [ "self._d.get", "self._d" ], "state_writes": [], "raises": [], "return_expressions": [ "self._d.get('parameters', {})" ], "awaits": false, "yields": false }, { "name": "_ToolDef", "qualname": "_ToolDef", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._ToolDef", "kind": "class", "signature": "class _ToolDef", "parameters": [ { "name": "d", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "_ToolDef", "docstring": "Wrap a tool definition dict for attribute access.", "summary": "Wrap a tool definition dict for attribute access.", "implementation": "Class `_ToolDef` declares 3 direct member(s).", "documented": true, "public": false, "addressable": true, "line": 102, "end_line": 117, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L102-L117", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "_ToolDef.__init__", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._ToolDef.__init__", "kind": "method", "signature": "def __init__(self, d: dict)", "parameters": [ { "name": "d", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `_ToolDef.__init__` updates `self._d`, `self._func`; calls `_FunctionDef`, `d.get`.", "implementation": "Method `_ToolDef.__init__` updates `self._d`, `self._func`; calls `_FunctionDef`, `d.get`.", "documented": false, "public": false, "addressable": true, "line": 107, "end_line": 109, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L107-L109", "decorators": [], "calls": [ "_FunctionDef", "d.get" ], "state_reads": [], "state_writes": [ "self._d", "self._func" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "type", "qualname": "_ToolDef.type", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._ToolDef.type", "kind": "method", "signature": "def type(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `_ToolDef.type` calls `self._d.get`; returns `self._d.get('type', 'function')`.", "implementation": "Method `_ToolDef.type` calls `self._d.get`; returns `self._d.get('type', 'function')`.", "documented": false, "public": false, "addressable": true, "line": 112, "end_line": 113, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L112-L113", "decorators": [ "property" ], "calls": [ "self._d.get" ], "state_reads": [ "self._d.get", "self._d" ], "state_writes": [], "raises": [], "return_expressions": [ "self._d.get('type', 'function')" ], "awaits": false, "yields": false }, { "name": "function", "qualname": "_ToolDef.function", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser._ToolDef.function", "kind": "method", "signature": "def function(self) -> _FunctionDef", "parameters": [], "return_annotation": "_FunctionDef", "docstring": "", "summary": "Method `_ToolDef.function` returns `self._func`.", "implementation": "Method `_ToolDef.function` returns `self._func`.", "documented": false, "public": false, "addressable": true, "line": 116, "end_line": 117, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L116-L117", "decorators": [ "property" ], "calls": [], "state_reads": [ "self._func" ], "state_writes": [], "raises": [], "return_expressions": [ "self._func" ], "awaits": false, "yields": false }, { "name": "StreamingXMLToolCallParser", "qualname": "StreamingXMLToolCallParser", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser", "kind": "class", "signature": "class StreamingXMLToolCallParser", "parameters": [], "return_annotation": "StreamingXMLToolCallParser", "docstring": "Streaming XML parser for Qwen 3.5 ```` format.\n\nArchitecture:\n 1. **Preprocessing** (``_preprocess_before_xml_parse``): scans raw text\n for ```` tags, extracts type hints from tool schemas,\n and rewrites the XML into expat-parseable form.\n 2. **Expat parsing**: an incremental ``xml.parsers.expat`` parser fires\n ``start_element`` / ``end_element`` / ``character_data`` callbacks.\n 3. **Type coercion** (``_coerce_param_value``): converts string values to\n int/float/bool/object/array based on JSON Schema type hints. Complex\n types use a deferred ``ast.literal_eval`` + ``json.loads`` fallback.\n 4. **Auto-closing**: if the model truncates output mid-tag, the parser\n synthesizes closing tags so partial tool calls are still extractable.\n\nStreaming: ``update(delta)`` feeds incremental text. Completed tool calls\nare emitted via ``get_streaming_output()`` as they close. State resets\nbetween tool calls within the same response.", "summary": "Streaming XML parser for Qwen 3.5 ```` format.", "implementation": "Class `StreamingXMLToolCallParser` declares 27 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 126, "end_line": 1427, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L126-L1427", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "StreamingXMLToolCallParser.__init__", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser.__init__", "kind": "method", "signature": "def __init__(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `StreamingXMLToolCallParser.__init__` updates `self.tools`, `self.tool_call_start_token`, `self.tool_call_end_token`, `self.function_start_token`; calls `self.reset_streaming_state`.", "implementation": "Method `StreamingXMLToolCallParser.__init__` updates `self.tools`, `self.tool_call_start_token`, `self.tool_call_end_token`, `self.function_start_token`; calls `self.reset_streaming_state`.", "documented": false, "public": false, "addressable": true, "line": 146, "end_line": 156, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L146-L156", "decorators": [], "calls": [ "self.reset_streaming_state" ], "state_reads": [ "self.reset_streaming_state" ], "state_writes": [ "self.tools", "self.tool_call_start_token", "self.tool_call_end_token", "self.function_start_token", "self.function_end_token", "self.parameter_start_token", "self.parameter_end_token" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "reset_streaming_state", "qualname": "StreamingXMLToolCallParser.reset_streaming_state", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser.reset_streaming_state", "kind": "method", "signature": "def reset_streaming_state(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Reset streaming parsing state", "summary": "Reset streaming parsing state", "implementation": "Method `StreamingXMLToolCallParser.reset_streaming_state` updates `self.deltas`, `self.tool_call_index`, `self.current_call_id`, `self.last_completed_call_id`; calls `ParserCreate`, `self.setup_parser`.", "documented": true, "public": true, "addressable": true, "line": 158, "end_line": 208, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L158-L208", "decorators": [], "calls": [ "ParserCreate", "self.setup_parser" ], "state_reads": [ "self.setup_parser" ], "state_writes": [ "self.deltas", "self.tool_call_index", "self.current_call_id", "self.last_completed_call_id", "self.current_function_name", "self.current_function_open", "self.implicit_tool_call_wrapper", "self._pending_implicit_delta", "self._pending_implicit_raw_text", "self._pending_implicit_text_buffer", "self._current_raw_element", "self.parameters", "self.current_param_name", "self.current_param_value", "self.current_param_value_converted", "self.current_param_is_first", "self.should_emit_end_newline", "self.start_quote_emitted", "self.streaming_buffer", "self.last_processed_pos", "self.text_content_buffer", "self._pre_inside_parameter", "self._pre_param_buffer", "self._pre_current_param_name", "self.defer_current_parameter", "self.deferred_param_raw_value", "self.parser" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "parse_single_streaming_chunks", "qualname": "StreamingXMLToolCallParser.parse_single_streaming_chunks", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser.parse_single_streaming_chunks", "kind": "method", "signature": "def parse_single_streaming_chunks(self, xml_chunk: str) -> DeltaMessage", "parameters": [ { "name": "xml_chunk", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Single XML chunk string" } ], "return_annotation": "DeltaMessage", "docstring": "Parse single streaming XML chunk and return Delta response\nThis is the actual streaming interface that receives chunks\none by one and maintains internal state\n\nArgs:\n xml_chunk: Single XML chunk string\nReturns:\n DeltaMessage: Contains delta information generated by this chunk,\n returns empty response if no complete elements", "summary": "Parse single streaming XML chunk and return Delta response This is the actual streaming interface that receives chunks one by one and maintains internal state Args: xml_chunk: Single XML chunk string Returns: DeltaMessage: Contains delta information generated by this chunk, returns empty response if no complete elements", "implementation": "Method `StreamingXMLToolCallParser.parse_single_streaming_chunks` updates `self.streaming_buffer`, `self.text_content_buffer`; calls `len`, `self._process_complete_xml_elements`, `xml_chunk.count`, `sum`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 210, "end_line": 330, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L210-L330", "decorators": [], "calls": [ "len", "self._process_complete_xml_elements", "xml_chunk.count", "sum", "isinstance", "self._end_element", "any", "logger.warning", "self._merge_new_deltas_to_single_response", "DeltaMessage", "self._emit_delta" ], "state_reads": [ "self.deltas", "self._process_complete_xml_elements", "self.current_call_id", "self.function_end_token", "self.current_param_name", "self._end_element", "self.current_function_name", "self.tool_call_end_token", "self._merge_new_deltas_to_single_response", "self.text_content_buffer", "self.tool_call_index", "self._emit_delta" ], "state_writes": [ "self.streaming_buffer", "self.text_content_buffer" ], "raises": [], "return_expressions": [ "result_delta", "text_delta", "DeltaMessage(content=None)" ], "awaits": false, "yields": false }, { "name": "_escape_xml_special_chars", "qualname": "StreamingXMLToolCallParser._escape_xml_special_chars", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._escape_xml_special_chars", "kind": "method", "signature": "def _escape_xml_special_chars(self, text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Original text" } ], "return_annotation": "str", "docstring": "Escape XML special characters\nArgs:\n text: Original text\nReturns:\n Escaped text", "summary": "Escape XML special characters Args: text: Original text Returns: Escaped text", "implementation": "Method `StreamingXMLToolCallParser._escape_xml_special_chars` calls `xml_escapes.items`, `text.replace`; returns `text`.", "documented": true, "public": false, "addressable": true, "line": 332, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L332-L351", "decorators": [], "calls": [ "xml_escapes.items", "text.replace" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "text" ], "awaits": false, "yields": false }, { "name": "_process_complete_xml_elements", "qualname": "StreamingXMLToolCallParser._process_complete_xml_elements", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._process_complete_xml_elements", "kind": "method", "signature": "def _process_complete_xml_elements(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Process complete XML elements in buffer\n\nReturns:\n bool: Whether complete elements were found and processed", "summary": "Process complete XML elements in buffer Returns: bool: Whether complete elements were found and processed", "implementation": "Method `StreamingXMLToolCallParser._process_complete_xml_elements` updates `self.last_processed_pos`, `self.text_content_buffer`, `self._current_raw_element`; calls `len`, `self._find_next_complete_element`, `self._should_skip_element`, `self._preprocess_xml_chunk`; returns `found_any`.", "documented": true, "public": false, "addressable": true, "line": 353, "end_line": 438, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L353-L438", "decorators": [], "calls": [ "len", "self._find_next_complete_element", "self._should_skip_element", "self._preprocess_xml_chunk", "preprocessed_element.strip().startswith", "preprocessed_element.strip", "DeltaMessage", "self._emit_delta", "self._end_element", "DeltaToolCall", "DeltaFunctionCall", "self._reset_xml_parser_after_tool_call", "self.parser.Parse", "logger.warning" ], "state_reads": [ "self.last_processed_pos", "self.streaming_buffer", "self._find_next_complete_element", "self._should_skip_element", "self._preprocess_xml_chunk", "self.tool_call_index", "self.text_content_buffer", "self._emit_delta", "self.current_call_id", "self.current_param_name", "self._end_element", "self.current_function_open", "self.current_function_name", "self._reset_xml_parser_after_tool_call", "self.parser.Parse", "self.parser" ], "state_writes": [ "self.last_processed_pos", "self.text_content_buffer", "self._current_raw_element" ], "raises": [], "return_expressions": [ "found_any" ], "awaits": false, "yields": false }, { "name": "_should_skip_element", "qualname": "StreamingXMLToolCallParser._should_skip_element", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._should_skip_element", "kind": "method", "signature": "def _should_skip_element(self, element: str) -> bool", "parameters": [ { "name": "element", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Element to evaluate" } ], "return_annotation": "bool", "docstring": "Determine whether an element should be skipped\n\nArgs:\n element: Element to evaluate\n\nReturns:\n bool: True means should skip, False means should process", "summary": "Determine whether an element should be skipped Args: element: Element to evaluate Returns: bool: True means should skip, False means should process", "implementation": "Method `StreamingXMLToolCallParser._should_skip_element` updates `self.text_content_buffer`; calls `element.startswith`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 440, "end_line": 474, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L440-L474", "decorators": [], "calls": [ "element.startswith" ], "state_reads": [ "self.tool_call_start_token", "self.function_start_token", "self.parameter_start_token", "self.current_call_id" ], "state_writes": [ "self.text_content_buffer" ], "raises": [], "return_expressions": [ "False", "True", "not element" ], "awaits": false, "yields": false }, { "name": "_looks_like_partial_tool_open", "qualname": "StreamingXMLToolCallParser._looks_like_partial_tool_open", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._looks_like_partial_tool_open", "kind": "method", "signature": "def _looks_like_partial_tool_open(self, fragment: str) -> bool", "parameters": [ { "name": "fragment", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "True if `fragment` could complete into a tool-related XML tag.\n\nCovers two shapes:\n * `fragment` is a prefix of a known tag head (e.g. ````.", "summary": "True if `fragment` could complete into a tool-related XML tag.", "implementation": "Method `StreamingXMLToolCallParser._looks_like_partial_tool_open` calls `fragment.startswith`, `prefix.startswith`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 488, "end_line": 501, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L488-L501", "decorators": [], "calls": [ "fragment.startswith", "prefix.startswith" ], "state_reads": [ "self._TOOL_TAG_PREFIXES" ], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "_find_next_complete_element", "qualname": "StreamingXMLToolCallParser._find_next_complete_element", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._find_next_complete_element", "kind": "method", "signature": "def _find_next_complete_element(self, start_pos: int) -> tuple[Optional[str], int]", "parameters": [ { "name": "start_pos", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Position to start searching" } ], "return_annotation": "tuple[Optional[str], int]", "docstring": "Find next complete XML element from specified position\n\nArgs:\n start_pos: Position to start searching\n\nReturns:\n (Complete element string, element end position),\n returns (None, start_pos) if no complete element found", "summary": "Find next complete XML element from specified position Args: start_pos: Position to start searching Returns: (Complete element string, element end position), returns (None, start_pos) if no complete element found", "implementation": "Method `StreamingXMLToolCallParser._find_next_complete_element` calls `buffer.startswith`, `buffer.find`, `self._looks_like_partial_tool_open`, `len`; has 6 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 503, "end_line": 569, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L503-L569", "decorators": [], "calls": [ "buffer.startswith", "buffer.find", "self._looks_like_partial_tool_open", "len" ], "state_reads": [ "self.streaming_buffer", "self._looks_like_partial_tool_open", "self.current_call_id" ], "state_writes": [], "raises": [], "return_expressions": [ "(None, start_pos)", "(buffer[:tag_end], start_pos + tag_end)", "(buffer[:tag_end2 + 1], start_pos + tag_end2 + 1)", "(buffer, start_pos + len(buffer))", "(text_content, start_pos + next_tag_pos)", "(remaining, start_pos + len(remaining))" ], "awaits": false, "yields": false }, { "name": "_merge_new_deltas_to_single_response", "qualname": "StreamingXMLToolCallParser._merge_new_deltas_to_single_response", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._merge_new_deltas_to_single_response", "kind": "method", "signature": "def _merge_new_deltas_to_single_response(self, initial_count: int) -> DeltaMessage", "parameters": [ { "name": "initial_count", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Delta count before processing" } ], "return_annotation": "DeltaMessage", "docstring": "Merge newly generated deltas from this processing\ninto a single DeltaMessage\n\nArgs:\n initial_count: Delta count before processing\n\nReturns:\n Merged DeltaMessage containing all newly generated delta information", "summary": "Merge newly generated deltas from this processing into a single DeltaMessage Args: initial_count: Delta count before processing Returns: Merged DeltaMessage containing all newly generated delta information", "implementation": "Method `StreamingXMLToolCallParser._merge_new_deltas_to_single_response` calls `len`, `DeltaMessage`, `merged_tool_calls.append`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 571, "end_line": 633, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L571-L633", "decorators": [], "calls": [ "len", "DeltaMessage", "merged_tool_calls.append" ], "state_reads": [ "self.deltas" ], "state_writes": [], "raises": [], "return_expressions": [ "DeltaMessage(content=None)", "new_deltas[0]", "DeltaMessage(content=merged_content if merged_content else None, tool_calls=merged_tool_calls)" ], "awaits": false, "yields": false }, { "name": "_preprocess_xml_chunk", "qualname": "StreamingXMLToolCallParser._preprocess_xml_chunk", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._preprocess_xml_chunk", "kind": "method", "signature": "def _preprocess_xml_chunk(self, chunk: str) -> str", "parameters": [ { "name": "chunk", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Original XML chunk" } ], "return_annotation": "str", "docstring": "Preprocess XML chunk, handle non-standard formats,\nand escape special characters\n\nArgs:\n chunk: Original XML chunk\n\nReturns:\n Processed XML chunk", "summary": "Preprocess XML chunk, handle non-standard formats, and escape special characters Args: chunk: Original XML chunk Returns: Processed XML chunk", "implementation": "Method `StreamingXMLToolCallParser._preprocess_xml_chunk` updates `self.defer_current_parameter`, `self.deferred_param_raw_value`, `self._pre_inside_parameter`, `self._pre_param_buffer`; calls `chunk.startswith`, `re.sub`, `processed.startswith`, `self._escape_xml_special_chars`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 635, "end_line": 748, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L635-L748", "decorators": [], "calls": [ "chunk.startswith", "re.sub", "processed.startswith", "self._escape_xml_special_chars", "self._get_param_type", "param_type.startswith", "re.match", "m.group" ], "state_reads": [ "self.tool_call_start_token", "self.tool_call_end_token", "self.function_start_token", "self.function_end_token", "self.parameter_start_token", "self.parameter_end_token", "self._pre_inside_parameter", "self._pre_param_buffer", "self._escape_xml_special_chars", "self._pre_current_param_name", "self._get_param_type" ], "state_writes": [ "self.defer_current_parameter", "self.deferred_param_raw_value", "self._pre_inside_parameter", "self._pre_param_buffer", "self._pre_current_param_name" ], "raises": [], "return_expressions": [ "f'{safe_text}'", "self._escape_xml_special_chars(original_chunk)", "''", "processed" ], "awaits": false, "yields": false }, { "name": "_emit_delta", "qualname": "StreamingXMLToolCallParser._emit_delta", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._emit_delta", "kind": "method", "signature": "def _emit_delta(self, delta: DeltaMessage)", "parameters": [ { "name": "delta", "kind": "positional or keyword", "annotation": "DeltaMessage", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Emit Delta response (streaming output)", "summary": "Emit Delta response (streaming output)", "implementation": "Method `StreamingXMLToolCallParser._emit_delta` calls `self.deltas.append`.", "documented": true, "public": false, "addressable": true, "line": 750, "end_line": 752, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L750-L752", "decorators": [], "calls": [ "self.deltas.append" ], "state_reads": [ "self.deltas.append", "self.deltas" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_auto_close_open_parameter_if_needed", "qualname": "StreamingXMLToolCallParser._auto_close_open_parameter_if_needed", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._auto_close_open_parameter_if_needed", "kind": "method", "signature": "def _auto_close_open_parameter_if_needed(self, incoming_tag: Optional[str]=None)", "parameters": [ { "name": "incoming_tag", "kind": "positional or keyword", "annotation": "Optional[str]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Before starting to process new elements,\nif there are unclosed tags from before,\nautomatically complete their endings to the parser.\n- If there are unclosed parameters,\nit's equivalent to feeding ``\n- When about to start a new function or tool_call,\nif there are unclosed functions, complete ``.\n- When about to start a new tool_call,\nif there are unclosed tool_calls, complete ``.", "summary": "Before starting to process new elements, if there are unclosed tags from before, automatically complete their endings to the parser.", "implementation": "Method `StreamingXMLToolCallParser._auto_close_open_parameter_if_needed` calls `self._end_element`.", "documented": true, "public": false, "addressable": true, "line": 754, "end_line": 777, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L754-L777", "decorators": [], "calls": [ "self._end_element" ], "state_reads": [ "self.current_param_name", "self._end_element", "self.current_function_name", "self.current_call_id" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_start_element", "qualname": "StreamingXMLToolCallParser._start_element", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._start_element", "kind": "method", "signature": "def _start_element(self, name: str, attrs: dict[str, str])", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attrs", "kind": "positional or keyword", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Handle XML start element events", "summary": "Handle XML start element events", "implementation": "Method `StreamingXMLToolCallParser._start_element` updates `self.parameters`, `self.current_call_id`, `self.current_param_is_first`, `self.tool_call_index`; calls `self._auto_close_open_parameter_if_needed`, `self._get_next_call_id`, `name.startswith`, `self._start_element`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 779, "end_line": 888, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L779-L888", "decorators": [], "calls": [ "self._auto_close_open_parameter_if_needed", "self._get_next_call_id", "name.startswith", "self._start_element", "self._extract_function_name", "DeltaMessage", "DeltaToolCall", "DeltaFunctionCall", "self._emit_delta", "self._flush_pending_implicit_delta", "self._extract_parameter_name" ], "state_reads": [ "self._auto_close_open_parameter_if_needed", "self._get_next_call_id", "self.current_call_id", "self._start_element", "self._extract_function_name", "self.tool_call_index", "self._current_raw_element", "self._emit_delta", "self._flush_pending_implicit_delta", "self._extract_parameter_name", "self.parameters" ], "state_writes": [ "self.parameters", "self.current_call_id", "self.current_param_is_first", "self.tool_call_index", "self.implicit_tool_call_wrapper", "self.current_function_name", "self.current_function_open", "self._pending_implicit_delta", "self._pending_implicit_raw_text", "self._pending_implicit_text_buffer", "self.current_param_name", "self.current_param_value", "self.current_param_value_converted", "self.start_quote_emitted" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_flush_pending_implicit_delta", "qualname": "StreamingXMLToolCallParser._flush_pending_implicit_delta", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._flush_pending_implicit_delta", "kind": "method", "signature": "def _flush_pending_implicit_delta(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Emit a deferred bare- delta now that the call is confirmed.", "summary": "Emit a deferred bare- delta now that the call is confirmed.", "implementation": "Method `StreamingXMLToolCallParser._flush_pending_implicit_delta` updates `self._pending_implicit_delta`, `self._pending_implicit_raw_text`, `self._pending_implicit_text_buffer`; calls `self._emit_delta`.", "documented": true, "public": false, "addressable": true, "line": 890, "end_line": 900, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L890-L900", "decorators": [], "calls": [ "self._emit_delta" ], "state_reads": [ "self._pending_implicit_delta", "self._emit_delta" ], "state_writes": [ "self._pending_implicit_delta", "self._pending_implicit_raw_text", "self._pending_implicit_text_buffer" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_abandon_pending_implicit_tool_call", "qualname": "StreamingXMLToolCallParser._abandon_pending_implicit_tool_call", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._abandon_pending_implicit_tool_call", "kind": "method", "signature": "def _abandon_pending_implicit_tool_call(self) -> tuple[str, str]", "parameters": [], "return_annotation": "tuple[str, str]", "docstring": "Roll back a deferred bare- auto-open: prose followed.\n\nDrops the pending function-name delta and unwinds the synthesised\nwrapper state so no tool_call is ever emitted for this fragment.\nReturns ``(raw_text, buffered_text)`` so the caller can restore the\noriginal prose (raw `` tag + any whitespace held while\nwaiting on commitment) as user-visible content.", "summary": "Roll back a deferred bare- auto-open: prose followed.", "implementation": "Method `StreamingXMLToolCallParser._abandon_pending_implicit_tool_call` updates `self._pending_implicit_delta`, `self._pending_implicit_raw_text`, `self._pending_implicit_text_buffer`, `self.implicit_tool_call_wrapper`; calls `self._reset_xml_parser_after_tool_call`; returns `(raw_text, buffered_text)`.", "documented": true, "public": false, "addressable": true, "line": 902, "end_line": 928, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L902-L928", "decorators": [], "calls": [ "self._reset_xml_parser_after_tool_call" ], "state_reads": [ "self._pending_implicit_raw_text", "self._pending_implicit_text_buffer", "self.tool_call_index", "self.current_call_id", "self._reset_xml_parser_after_tool_call" ], "state_writes": [ "self._pending_implicit_delta", "self._pending_implicit_raw_text", "self._pending_implicit_text_buffer", "self.implicit_tool_call_wrapper", "self.current_function_name", "self.current_function_open", "self.tool_call_index", "self.last_completed_call_id", "self.current_call_id" ], "raises": [], "return_expressions": [ "(raw_text, buffered_text)" ], "awaits": false, "yields": false }, { "name": "_char_data", "qualname": "StreamingXMLToolCallParser._char_data", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._char_data", "kind": "method", "signature": "def _char_data(self, data: str)", "parameters": [ { "name": "data", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Handle XML character data events", "summary": "Handle XML character data events", "implementation": "Method `StreamingXMLToolCallParser._char_data` updates `self._pending_implicit_text_buffer`, `self.should_emit_end_newline`, `self.current_param_value`, `self.start_quote_emitted`; calls `data.strip`, `self._abandon_pending_implicit_tool_call`, `self._emit_delta`, `DeltaMessage`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 930, "end_line": 1025, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L930-L1025", "decorators": [], "calls": [ "data.strip", "self._abandon_pending_implicit_tool_call", "self._emit_delta", "DeltaMessage", "original_data.endswith", "self._get_param_type", "data.startswith", "DeltaToolCall", "DeltaFunctionCall", "self._convert_param_value", "self._convert_for_json_streaming", "len" ], "state_reads": [ "self._pending_implicit_delta", "self.current_param_name", "self._abandon_pending_implicit_tool_call", "self._emit_delta", "self.defer_current_parameter", "self.should_emit_end_newline", "self._get_param_type", "self.current_param_value", "self.start_quote_emitted", "self.tool_call_index", "self.current_call_id", "self._convert_param_value", "self._convert_for_json_streaming", "self.current_param_value_converted" ], "state_writes": [ "self._pending_implicit_text_buffer", "self.should_emit_end_newline", "self.current_param_value", "self.start_quote_emitted", "self.current_param_value_converted" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_end_element", "qualname": "StreamingXMLToolCallParser._end_element", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._end_element", "kind": "method", "signature": "def _end_element(self, name: str)", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Handle XML end element events", "summary": "Handle XML end element events", "implementation": "Method `StreamingXMLToolCallParser._end_element` updates `self.should_emit_end_newline`, `self.current_param_name`, `self.current_param_value`, `self.current_param_value_converted`; calls `name.startswith`, `self._auto_close_open_parameter_if_needed`, `ast.literal_eval`, `json.dumps`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 1027, "end_line": 1206, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1027-L1206", "decorators": [], "calls": [ "name.startswith", "self._auto_close_open_parameter_if_needed", "ast.literal_eval", "json.dumps", "DeltaMessage", "DeltaToolCall", "DeltaFunctionCall", "self._emit_delta", "self._get_param_type", "self._convert_param_value", "self._flush_pending_implicit_delta", "self._end_element", "self.text_content_buffer.strip", "self._reset_xml_parser_after_tool_call" ], "state_reads": [ "self.current_param_name", "self._auto_close_open_parameter_if_needed", "self.current_param_value", "self.defer_current_parameter", "self.deferred_param_raw_value", "self.should_emit_end_newline", "self.tool_call_index", "self.current_call_id", "self._emit_delta", "self.parameters", "self._get_param_type", "self._convert_param_value", "self.start_quote_emitted", "self._flush_pending_implicit_delta", "self.implicit_tool_call_wrapper", "self._end_element", "self.current_function_open", "self.text_content_buffer.strip", "self.text_content_buffer", "self._reset_xml_parser_after_tool_call" ], "state_writes": [ "self.should_emit_end_newline", "self.current_param_name", "self.current_param_value", "self.current_param_value_converted", "self.start_quote_emitted", "self.defer_current_parameter", "self.deferred_param_raw_value", "self.current_function_open", "self.implicit_tool_call_wrapper" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "setup_parser", "qualname": "StreamingXMLToolCallParser.setup_parser", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser.setup_parser", "kind": "method", "signature": "def setup_parser(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Set up XML parser event handlers", "summary": "Set up XML parser event handlers", "implementation": "Method `StreamingXMLToolCallParser.setup_parser` updates `self.parser.buffer_text`, `self.parser.StartElementHandler`, `self.parser.EndElementHandler`, `self.parser.CharacterDataHandler`.", "documented": true, "public": true, "addressable": true, "line": 1208, "end_line": 1213, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1208-L1213", "decorators": [], "calls": [], "state_reads": [ "self.parser", "self._start_element", "self._end_element", "self._char_data" ], "state_writes": [ "self.parser.buffer_text", "self.parser.StartElementHandler", "self.parser.EndElementHandler", "self.parser.CharacterDataHandler" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "set_tools", "qualname": "StreamingXMLToolCallParser.set_tools", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser.set_tools", "kind": "method", "signature": "def set_tools(self, tools: Union[list[ChatCompletionToolsParam], None])", "parameters": [ { "name": "tools", "kind": "positional or keyword", "annotation": "Union[list[ChatCompletionToolsParam], None]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Set tool configuration information", "summary": "Set tool configuration information", "implementation": "Method `StreamingXMLToolCallParser.set_tools` updates `self.tools`.", "documented": true, "public": true, "addressable": true, "line": 1215, "end_line": 1217, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1215-L1217", "decorators": [], "calls": [], "state_reads": [], "state_writes": [ "self.tools" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_get_next_call_id", "qualname": "StreamingXMLToolCallParser._get_next_call_id", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._get_next_call_id", "kind": "method", "signature": "def _get_next_call_id(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Generate unique call ID", "summary": "Generate unique call ID", "implementation": "Method `StreamingXMLToolCallParser._get_next_call_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:24]}'`.", "documented": true, "public": false, "addressable": true, "line": 1219, "end_line": 1221, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1219-L1221", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:24]}'" ], "awaits": false, "yields": false }, { "name": "_extract_function_name", "qualname": "StreamingXMLToolCallParser._extract_function_name", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._extract_function_name", "kind": "method", "signature": "def _extract_function_name(self, name: str, attrs: dict[str, str]) -> Optional[str]", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attrs", "kind": "positional or keyword", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[str]", "docstring": "Extract function name from various formats", "summary": "Extract function name from various formats", "implementation": "Method `StreamingXMLToolCallParser._extract_function_name` calls `name.split`, `len`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1223, "end_line": 1233, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1223-L1233", "decorators": [], "calls": [ "name.split", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "attrs['name']", "parts[1]", "None" ], "awaits": false, "yields": false }, { "name": "_extract_parameter_name", "qualname": "StreamingXMLToolCallParser._extract_parameter_name", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._extract_parameter_name", "kind": "method", "signature": "def _extract_parameter_name(self, name: str, attrs: dict[str, str]) -> Optional[str]", "parameters": [ { "name": "name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attrs", "kind": "positional or keyword", "annotation": "dict[str, str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[str]", "docstring": "Extract parameter name from various formats", "summary": "Extract parameter name from various formats", "implementation": "Method `StreamingXMLToolCallParser._extract_parameter_name` calls `name.split`, `len`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1235, "end_line": 1247, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1235-L1247", "decorators": [], "calls": [ "name.split", "len" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "attrs['name']", "parts[1]", "None" ], "awaits": false, "yields": false }, { "name": "_get_param_type", "qualname": "StreamingXMLToolCallParser._get_param_type", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._get_param_type", "kind": "method", "signature": "def _get_param_type(self, param_name: str) -> str", "parameters": [ { "name": "param_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Parameter name" } ], "return_annotation": "str", "docstring": "Get parameter type based on tool configuration, defaults to string\nArgs:\n param_name: Parameter name\n\nReturns:\n Parameter type", "summary": "Get parameter type based on tool configuration, defaults to string Args: param_name: Parameter name Returns: Parameter type", "implementation": "Method `StreamingXMLToolCallParser._get_param_type` calls `hasattr`, `isinstance`, `self.repair_param_type`, `str`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1249, "end_line": 1287, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1249-L1287", "decorators": [], "calls": [ "hasattr", "isinstance", "self.repair_param_type", "str", "properties[param_name].get", "param_config.get" ], "state_reads": [ "self.tools", "self.current_function_name", "self.repair_param_type" ], "state_writes": [], "raises": [], "return_expressions": [ "'string'", "self.repair_param_type(str(properties[param_name].get('type', 'string')))", "self.repair_param_type(str(param_config.get('type', 'string')))" ], "awaits": false, "yields": false }, { "name": "repair_param_type", "qualname": "StreamingXMLToolCallParser.repair_param_type", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser.repair_param_type", "kind": "method", "signature": "def repair_param_type(self, param_type: str) -> str", "parameters": [ { "name": "param_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Parameter type" } ], "return_annotation": "str", "docstring": "Repair unknown parameter types by treating them as string\nArgs:\n param_type: Parameter type\n\nReturns:\n Repaired parameter type", "summary": "Repair unknown parameter types by treating them as string Args: param_type: Parameter type Returns: Repaired parameter type", "implementation": "Method `StreamingXMLToolCallParser.repair_param_type` calls `param_type.startswith`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1289, "end_line": 1315, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1289-L1315", "decorators": [], "calls": [ "param_type.startswith" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "param_type", "'string'" ], "awaits": false, "yields": false }, { "name": "_convert_param_value", "qualname": "StreamingXMLToolCallParser._convert_param_value", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._convert_param_value", "kind": "method", "signature": "def _convert_param_value(self, param_value: str, param_type: str) -> Any", "parameters": [ { "name": "param_value", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Parameter value" }, { "name": "param_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Parameter type" } ], "return_annotation": "Any", "docstring": "Convert value based on parameter type\nArgs:\n param_value: Parameter value\n param_type: Parameter type\n\nReturns:\n Converted value", "summary": "Convert value based on parameter type Args: param_value: Parameter value param_type: Parameter type Returns: Converted value", "implementation": "Method `StreamingXMLToolCallParser._convert_param_value` calls `param_value.lower`, `param_type.strip().lower`, `param_type.strip`, `param_type.startswith`; has 5 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1317, "end_line": 1371, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1317-L1371", "decorators": [], "calls": [ "param_value.lower", "param_type.strip().lower", "param_type.strip", "param_type.startswith", "int", "logger.warning", "float" ], "state_reads": [ "self.current_param_name", "self.current_function_name" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "param_value", "int(param_value)", "float_param_value if float_param_value - int(float_param_value) != 0 else int(float_param_value)", "param_value == 'true'" ], "awaits": false, "yields": false }, { "name": "_convert_for_json_streaming", "qualname": "StreamingXMLToolCallParser._convert_for_json_streaming", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._convert_for_json_streaming", "kind": "method", "signature": "def _convert_for_json_streaming(self, converted_value: Any, param_type: str) -> str", "parameters": [ { "name": "converted_value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Converted value" }, { "name": "param_type", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Parameter type" } ], "return_annotation": "str", "docstring": "Convert converted_value based on\nwhether it's empty and if type is string\nArgs:\n converted_value: Converted value\n param_type: Parameter type\n\nReturns:\n Converted string for streaming output", "summary": "Convert converted_value based on whether it's empty and if type is string Args: converted_value: Converted value param_type: Parameter type Returns: Converted string for streaming output", "implementation": "Method `StreamingXMLToolCallParser._convert_for_json_streaming` calls `json.dumps`, `isinstance`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1373, "end_line": 1395, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1373-L1395", "decorators": [], "calls": [ "json.dumps", "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''", "json.dumps(converted_value, ensure_ascii=False)[1:-1]", "json.dumps(converted_value, ensure_ascii=False)", "converted_value" ], "awaits": false, "yields": false }, { "name": "_reset_xml_parser_after_tool_call", "qualname": "StreamingXMLToolCallParser._reset_xml_parser_after_tool_call", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.StreamingXMLToolCallParser._reset_xml_parser_after_tool_call", "kind": "method", "signature": "def _reset_xml_parser_after_tool_call(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Each tool_call is treated as a separate XML document,\nso we need to reset the parser after each tool_call.", "summary": "Each tool_call is treated as a separate XML document, so we need to reset the parser after each tool_call.", "implementation": "Method `StreamingXMLToolCallParser._reset_xml_parser_after_tool_call` updates `self.parser`, `self.last_completed_call_id`, `self.current_call_id`, `self.current_function_name`; calls `ParserCreate`, `self.setup_parser`.", "documented": true, "public": false, "addressable": true, "line": 1397, "end_line": 1427, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1397-L1427", "decorators": [], "calls": [ "ParserCreate", "self.setup_parser" ], "state_reads": [ "self.setup_parser", "self.current_call_id" ], "state_writes": [ "self.parser", "self.last_completed_call_id", "self.current_call_id", "self.current_function_name", "self.current_function_open", "self.parameters", "self.current_param_name", "self.current_param_value", "self.current_param_value_converted", "self.current_param_is_first", "self.should_emit_end_newline", "self.start_quote_emitted", "self.text_content_buffer", "self._pre_inside_parameter", "self._pre_param_buffer", "self._pre_current_param_name", "self.defer_current_parameter", "self.deferred_param_raw_value" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "Qwen3XMLToolParser", "qualname": "Qwen3XMLToolParser", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.Qwen3XMLToolParser", "kind": "class", "signature": "class Qwen3XMLToolParser(ToolParser)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "Qwen3XMLToolParser", "docstring": "XML tool call parser for Qwen 3.5 models, adapted for vllm-mlx.\n\nCore parsing logic from vLLM PR #25028 (Qwen API team).\nUses expat-based streaming XML parser with type coercion,\ndeferred parsing for complex types, and auto-closing of\nmalformed XML.", "summary": "XML tool call parser for Qwen 3.5 models, adapted for vllm-mlx.", "implementation": "Class `Qwen3XMLToolParser` derives from `ToolParser` and declares 4 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 1442, "end_line": 1559, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1442-L1559", "decorators": [ "ToolParserManager.register_module(['qwen3_xml', 'qwen3.5', 'qwen3_coder'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "Qwen3XMLToolParser.__init__", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.Qwen3XMLToolParser.__init__", "kind": "method", "signature": "def __init__(self, tokenizer=None)", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Method `Qwen3XMLToolParser.__init__` updates `self._xml_parser`; calls `super().__init__`, `super`, `StreamingXMLToolCallParser`, `logger.info`.", "implementation": "Method `Qwen3XMLToolParser.__init__` updates `self._xml_parser`; calls `super().__init__`, `super`, `StreamingXMLToolCallParser`, `logger.info`.", "documented": false, "public": false, "addressable": true, "line": 1454, "end_line": 1460, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1454-L1460", "decorators": [], "calls": [ "super().__init__", "super", "StreamingXMLToolCallParser", "logger.info" ], "state_reads": [ "self.__class__.__name__", "self.__class__" ], "state_writes": [ "self._xml_parser" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_wrap_tools", "qualname": "Qwen3XMLToolParser._wrap_tools", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.Qwen3XMLToolParser._wrap_tools", "kind": "method", "signature": "def _wrap_tools(request: dict[str, Any] | None) -> list[_ToolDef] | None", "parameters": [ { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[_ToolDef] | None", "docstring": "Convert tool definition dicts to _ToolDef wrappers for attribute access.", "summary": "Convert tool definition dicts to _ToolDef wrappers for attribute access.", "implementation": "Method `Qwen3XMLToolParser._wrap_tools` calls `request.get`, `_ToolDef`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 1463, "end_line": 1467, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1463-L1467", "decorators": [ "staticmethod" ], "calls": [ "request.get", "_ToolDef" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[_ToolDef(t) for t in request['tools']]", "None" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "Qwen3XMLToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.Qwen3XMLToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from complete Qwen 3.5 output.", "summary": "Extract tool calls from complete Qwen 3.5 output.", "implementation": "Method `Qwen3XMLToolParser.extract_tool_calls` calls `self.strip_think_tags`, `self._xml_parser.reset_streaming_state`, `self._wrap_tools`, `self._xml_parser.set_tools`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1469, "end_line": 1507, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1469-L1507", "decorators": [], "calls": [ "self.strip_think_tags", "self._xml_parser.reset_streaming_state", "self._wrap_tools", "self._xml_parser.set_tools", "self._xml_parser.parse_single_streaming_chunks", "ExtractedToolCallInformation", "tool_calls.append", "uuid.uuid4", "len" ], "state_reads": [ "self.strip_think_tags", "self._xml_parser.reset_streaming_state", "self._xml_parser", "self._wrap_tools", "self._xml_parser.set_tools", "self._xml_parser.parse_single_streaming_chunks" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=result.content if result.content else model_out…", "ExtractedToolCallInformation(tools_called=len(tool_calls) > 0, tool_calls=tool_calls, content=result.content)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "Qwen3XMLToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.qwen3_xml_tool_parser.Qwen3XMLToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Qwen 3.5 output.\n\nReturns dict with 'tool_calls' and/or 'content' keys,\nor None to suppress the chunk.", "summary": "Extract tool calls from streaming Qwen 3.5 output.", "implementation": "Method `Qwen3XMLToolParser.extract_tool_calls_streaming` calls `self._xml_parser.reset_streaming_state`, `self._wrap_tools`, `self._xml_parser.set_tools`, `self._xml_parser.parse_single_streaming_chunks`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 1509, "end_line": 1559, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen3_xml_tool_parser.py#L1509-L1559", "decorators": [], "calls": [ "self._xml_parser.reset_streaming_state", "self._wrap_tools", "self._xml_parser.set_tools", "self._xml_parser.parse_single_streaming_chunks", "tool_calls.append" ], "state_reads": [ "self._xml_parser.reset_streaming_state", "self._xml_parser", "self._wrap_tools", "self._xml_parser.set_tools", "self._xml_parser.parse_single_streaming_chunks" ], "state_writes": [], "raises": [], "return_expressions": [ "None", "{'tool_calls': tool_calls}", "{'content': result.content}" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.qwen_tool_parser", "path": "vllm_mlx/tool_parsers/qwen_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/qwen_tool_parser.md", "docstring": "Qwen tool call parser for vllm-mlx.\n\nHandles Qwen's tool calling formats:\n- XML style: {\"name\": \"func\", \"arguments\": {...}}\n- Bracket style: [Calling tool: func_name({\"arg\": \"value\"})]\n- Function style: value", "summary": "Qwen tool call parser for vllm-mlx.", "line_count": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L1-L351", "members": [ "_parse_param_value", "generate_tool_id", "QwenToolParser" ], "symbols": [ { "name": "_parse_param_value", "qualname": "_parse_param_value", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser._parse_param_value", "kind": "function", "signature": "def _parse_param_value(val: str) -> Any", "parameters": [ { "name": "val", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Parse a parameter value, handling JSON literals and plain strings.", "summary": "Parse a parameter value, handling JSON literals and plain strings.", "implementation": "Function `_parse_param_value` calls `json.loads`, `ast.literal_eval`, `isinstance`, `sorted`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 25, "end_line": 40, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L25-L40", "decorators": [], "calls": [ "json.loads", "ast.literal_eval", "isinstance", "sorted", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "json.loads(val)", "val", "python_val" ], "awaits": false, "yields": false }, { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 43, "end_line": 45, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L43-L45", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "QwenToolParser", "qualname": "QwenToolParser", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser.QwenToolParser", "kind": "class", "signature": "class QwenToolParser(ToolParser)", "parameters": [], "return_annotation": "QwenToolParser", "docstring": "Tool call parser for Qwen models.\n\nSupports multiple Qwen tool call formats:\n- XML: {\"name\": \"func\", \"arguments\": {...}}\n- Bracket: [Calling tool: func_name({\"arg\": \"value\"})]\n- Function: value\n\nUsed when --enable-auto-tool-choice --tool-call-parser qwen are set.", "summary": "Tool call parser for Qwen models.", "implementation": "Class `QwenToolParser` derives from `ToolParser` and declares 6 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 49, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L49-L351", "decorators": [ "ToolParserManager.register_module(['qwen', 'qwen3'])" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "QwenToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser.QwenToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from a complete Qwen model response.", "summary": "Extract tool calls from a complete Qwen model response.", "implementation": "Method `QwenToolParser.extract_tool_calls` calls `self.strip_think_tags`, `self.BRACKET_PATTERN.findall`, `json.loads`, `tool_calls.append`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 78, "end_line": 197, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L78-L197", "decorators": [], "calls": [ "self.strip_think_tags", "self.BRACKET_PATTERN.findall", "json.loads", "tool_calls.append", "generate_tool_id", "name.strip", "isinstance", "json.dumps", "str", "self.BRACKET_PATTERN.sub('', cleaned_text).strip", "self.BRACKET_PATTERN.sub", "self.XML_PATTERN.findall", "data.get", "self.XML_PATTERN.sub('', cleaned_text).strip", "self.XML_PATTERN.sub", "self.FUNCTION_PATTERN.findall", "params_block.strip", "params_block_stripped.startswith", "self.PARAM_PATTERN.findall", "p_name.strip", "_parse_param_value", "p_value.strip", "self.FUNCTION_PATTERN.sub('', cleaned_text).strip", "self.FUNCTION_PATTERN.sub", "self.EMPTY_TOOL_CALL.sub('', cleaned_text).strip", "self.EMPTY_TOOL_CALL.sub", "self._strip_unclosed_markup", "ExtractedToolCallInformation" ], "state_reads": [ "self.strip_think_tags", "self.BRACKET_PATTERN.findall", "self.BRACKET_PATTERN", "self.BRACKET_PATTERN.sub", "self.XML_PATTERN.findall", "self.XML_PATTERN", "self.XML_PATTERN.sub", "self.FUNCTION_PATTERN.findall", "self.FUNCTION_PATTERN", "self.PARAM_PATTERN.findall", "self.PARAM_PATTERN", "self.FUNCTION_PATTERN.sub", "self.EMPTY_TOOL_CALL.sub", "self.EMPTY_TOOL_CALL", "self._strip_unclosed_markup" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=cleaned_text if cleaned_text else None)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=stripped)" ], "awaits": false, "yields": false }, { "name": "_strip_unclosed_markup", "qualname": "QwenToolParser._strip_unclosed_markup", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser.QwenToolParser._strip_unclosed_markup", "kind": "method", "signature": "def _strip_unclosed_markup(text: str) -> str", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Strip a trailing unclosed tool-call marker (truncated output).\n\nWhen generation hits max_tokens mid-tool-call, a partial\n````/`` bool", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if text ends with an incomplete tool call marker prefix.", "summary": "Check if text ends with an incomplete tool call marker prefix.", "implementation": "Method `QwenToolParser._has_partial_marker` calls `self._get_partial_marker_len`; returns `self._get_partial_marker_len(text) > 0`.", "documented": true, "public": false, "addressable": true, "line": 235, "end_line": 237, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L235-L237", "decorators": [], "calls": [ "self._get_partial_marker_len" ], "state_reads": [ "self._get_partial_marker_len" ], "state_writes": [], "raises": [], "return_expressions": [ "self._get_partial_marker_len(text) > 0" ], "awaits": false, "yields": false }, { "name": "_get_partial_marker_len", "qualname": "QwenToolParser._get_partial_marker_len", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser.QwenToolParser._get_partial_marker_len", "kind": "method", "signature": "def _get_partial_marker_len(self, text: str) -> int", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int", "docstring": "Return the length of a partial tool call marker suffix at end of text.", "summary": "Return the length of a partial tool call marker suffix at end of text.", "implementation": "Method `QwenToolParser._get_partial_marker_len` calls `range`, `len`, `tail.endswith`; returns `best`.", "documented": true, "public": false, "addressable": true, "line": 239, "end_line": 248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L239-L248", "decorators": [], "calls": [ "range", "len", "tail.endswith" ], "state_reads": [ "self._PARTIAL_MARKERS" ], "state_writes": [], "raises": [], "return_expressions": [ "best" ], "awaits": false, "yields": false }, { "name": "_was_buffering", "qualname": "QwenToolParser._was_buffering", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser.QwenToolParser._was_buffering", "kind": "method", "signature": "def _was_buffering(self, previous_text: str) -> bool", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if the previous call was buffering a partial marker.", "summary": "Check if the previous call was buffering a partial marker.", "implementation": "Method `QwenToolParser._was_buffering` calls `self._has_partial_marker`; returns `self._has_partial_marker(previous_text)`.", "documented": true, "public": false, "addressable": true, "line": 250, "end_line": 252, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L250-L252", "decorators": [], "calls": [ "self._has_partial_marker" ], "state_reads": [ "self._has_partial_marker" ], "state_writes": [], "raises": [], "return_expressions": [ "self._has_partial_marker(previous_text)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "QwenToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.qwen_tool_parser.QwenToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming Qwen model output.", "summary": "Extract tool calls from streaming Qwen model output.", "implementation": "Method `QwenToolParser.extract_tool_calls_streaming` calls `self._has_partial_marker`, `self._get_partial_marker_len`, `len`, `self._was_buffering`; has 6 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 254, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/qwen_tool_parser.py#L254-L351", "decorators": [], "calls": [ "self._has_partial_marker", "self._get_partial_marker_len", "len", "self._was_buffering", "range", "previous_text.endswith", "current_text.count", "previous_text.count", "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self._has_partial_marker", "self._get_partial_marker_len", "self._was_buffering", "self._PARTIAL_MARKERS", "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text[:safe_chars]}", "None", "{'content': prefix + delta_text}", "{'content': delta_text}", "{'tool_calls': [{'index': prev_func_close + i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'ar…", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.tool_parsers.xlam_tool_parser", "path": "vllm_mlx/tool_parsers/xlam_tool_parser.py", "page_path": "reference/api/vllm_mlx/tool_parsers/xlam_tool_parser.md", "docstring": "xLAM tool call parser for vllm-mlx.\n\nHandles Salesforce xLAM models' tool calling format which supports:\n- JSON arrays of tool calls\n- Tool calls in markdown code blocks\n- Tool calls after reasoning blocks", "summary": "xLAM tool call parser for vllm-mlx.", "line_count": 177, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/xlam_tool_parser.py#L1-L177", "members": [ "generate_tool_id", "xLAMToolParser" ], "symbols": [ { "name": "generate_tool_id", "qualname": "generate_tool_id", "full_name": "vllm_mlx.tool_parsers.xlam_tool_parser.generate_tool_id", "kind": "function", "signature": "def generate_tool_id() -> str", "parameters": [], "return_annotation": "str", "docstring": "Generate a unique tool call ID.", "summary": "Generate a unique tool call ID.", "implementation": "Function `generate_tool_id` calls `uuid.uuid4`; returns `f'call_{uuid.uuid4().hex[:8]}'`.", "documented": true, "public": true, "addressable": true, "line": 24, "end_line": 26, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/xlam_tool_parser.py#L24-L26", "decorators": [], "calls": [ "uuid.uuid4" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'call_{uuid.uuid4().hex[:8]}'" ], "awaits": false, "yields": false }, { "name": "xLAMToolParser", "qualname": "xLAMToolParser", "full_name": "vllm_mlx.tool_parsers.xlam_tool_parser.xLAMToolParser", "kind": "class", "signature": "class xLAMToolParser(ToolParser)", "parameters": [], "return_annotation": "xLAMToolParser", "docstring": "Tool call parser for Salesforce xLAM models.\n\nSupports multiple formats:\n- JSON array: [{\"name\": \"func\", \"arguments\": {...}}]\n- Markdown code blocks: ```json [...] ```\n- After thinking: [...]\n\nUsed when --enable-auto-tool-choice --tool-call-parser xlam are set.", "summary": "Tool call parser for Salesforce xLAM models.", "implementation": "Class `xLAMToolParser` derives from `ToolParser` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 30, "end_line": 177, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/xlam_tool_parser.py#L30-L177", "decorators": [ "ToolParserManager.register_module('xlam')" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_try_extract_json", "qualname": "xLAMToolParser._try_extract_json", "full_name": "vllm_mlx.tool_parsers.xlam_tool_parser.xLAMToolParser._try_extract_json", "kind": "method", "signature": "def _try_extract_json(self, text: str) -> tuple[str | None, list | None]", "parameters": [ { "name": "text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "tuple[str | None, list | None]", "docstring": "Try to extract JSON tool calls from text.\n\nReturns:\n Tuple of (content, tool_calls_list)", "summary": "Try to extract JSON tool calls from text.", "implementation": "Method `xLAMToolParser._try_extract_json` calls `pattern.findall`, `json.loads`, `match.strip`, `isinstance`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 47, "end_line": 91, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/xlam_tool_parser.py#L47-L91", "decorators": [], "calls": [ "pattern.findall", "json.loads", "match.strip", "isinstance", "pattern.sub('', text).strip", "pattern.sub", "self.THINKING_PATTERN.search", "thinking_match.group(1).strip", "thinking_match.group", "text[:thinking_match.start() + len('')].strip", "thinking_match.start", "len", "text.strip", "text.startswith" ], "state_reads": [ "self.CODE_BLOCK_PATTERN", "self.TOOL_CALLS_TAG_PATTERN", "self.THINKING_PATTERN.search", "self.THINKING_PATTERN" ], "state_writes": [], "raises": [], "return_expressions": [ "(content if content else None, parsed)", "(None, parsed)", "(text, None)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls", "qualname": "xLAMToolParser.extract_tool_calls", "full_name": "vllm_mlx.tool_parsers.xlam_tool_parser.xLAMToolParser.extract_tool_calls", "kind": "method", "signature": "def extract_tool_calls(self, model_output: str, request: dict[str, Any] | None=None) -> ExtractedToolCallInformation", "parameters": [ { "name": "model_output", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "ExtractedToolCallInformation", "docstring": "Extract tool calls from xLAM model output.", "summary": "Extract tool calls from xLAM model output.", "implementation": "Method `xLAMToolParser.extract_tool_calls` calls `self._try_extract_json`, `ExtractedToolCallInformation`, `isinstance`, `call.get`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 93, "end_line": 131, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/xlam_tool_parser.py#L93-L131", "decorators": [], "calls": [ "self._try_extract_json", "ExtractedToolCallInformation", "isinstance", "call.get", "tool_calls.append", "generate_tool_id", "json.dumps", "str" ], "state_reads": [ "self._try_extract_json" ], "state_writes": [], "raises": [], "return_expressions": [ "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=content or model_output)", "ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=content)", "ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output)" ], "awaits": false, "yields": false }, { "name": "extract_tool_calls_streaming", "qualname": "xLAMToolParser.extract_tool_calls_streaming", "full_name": "vllm_mlx.tool_parsers.xlam_tool_parser.xLAMToolParser.extract_tool_calls_streaming", "kind": "method", "signature": "def extract_tool_calls_streaming(self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int] | None=None, current_token_ids: Sequence[int] | None=None, delta_token_ids: Sequence[int] | None=None, request: dict[str, Any] | None=None) -> dict[str, Any] | None", "parameters": [ { "name": "previous_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "current_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "delta_text", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "previous_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "current_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "delta_token_ids", "kind": "positional or keyword", "annotation": "Sequence[int] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "request", "kind": "positional or keyword", "annotation": "dict[str, Any] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "dict[str, Any] | None", "docstring": "Extract tool calls from streaming xLAM model output.", "summary": "Extract tool calls from streaming xLAM model output.", "implementation": "Method `xLAMToolParser.extract_tool_calls_streaming` calls `any`, `current_text.strip`, `stripped.startswith`, `self.extract_tool_calls`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 133, "end_line": 177, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/tool_parsers/xlam_tool_parser.py#L133-L177", "decorators": [], "calls": [ "any", "current_text.strip", "stripped.startswith", "self.extract_tool_calls", "enumerate" ], "state_reads": [ "self.extract_tool_calls" ], "state_writes": [], "raises": [], "return_expressions": [ "{'content': delta_text}", "{'tool_calls': [{'index': i, 'id': tc['id'], 'type': 'function', 'function': {'name': tc['name'], 'arguments': tc['argu…", "None" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.utils", "path": "vllm_mlx/utils/__init__.py", "page_path": "reference/api/vllm_mlx/utils/index.md", "docstring": "Utility modules for vllm-mlx.", "summary": "Utility modules for vllm-mlx.", "line_count": 13, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/__init__.py#L1-L13", "members": [ "__all__" ], "symbols": [] }, { "name": "vllm_mlx.utils.chat_templates", "path": "vllm_mlx/utils/chat_templates.py", "page_path": "reference/api/vllm_mlx/utils/chat_templates.md", "docstring": "Chat templates for various models.\n\nThis module contains Jinja2 chat templates for models that don't include\nthem in their tokenizer configuration.", "summary": "Chat templates for various models.", "line_count": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/chat_templates.py#L1-L225", "members": [ "NEMOTRON_CHAT_TEMPLATE", "DEFAULT_CHATML_TEMPLATE" ], "symbols": [] }, { "name": "vllm_mlx.utils.download", "path": "vllm_mlx/utils/download.py", "page_path": "reference/api/vllm_mlx/utils/download.md", "docstring": "Resumable model download with retry/timeout support.\n\nPre-downloads models via huggingface_hub.snapshot_download() with\nconfigurable timeout and retry logic before passing to mlx-lm/mlx-vlm.", "summary": "Resumable model download with retry/timeout support.", "line_count": 144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/download.py#L1-L144", "members": [ "logger", "LLM_ALLOW_PATTERNS", "MLLM_ALLOW_PATTERNS", "DownloadConfig", "ensure_model_downloaded" ], "symbols": [ { "name": "DownloadConfig", "qualname": "DownloadConfig", "full_name": "vllm_mlx.utils.download.DownloadConfig", "kind": "class", "signature": "class DownloadConfig", "parameters": [ { "name": "download_timeout", "kind": "field", "annotation": "int", "default": "300", "required": false, "description": "Optional constructor field; defaults to `300`." }, { "name": "max_retries", "kind": "field", "annotation": "int", "default": "3", "required": false, "description": "Optional constructor field; defaults to `3`." }, { "name": "retry_backoff_base", "kind": "field", "annotation": "float", "default": "2.0", "required": false, "description": "Optional constructor field; defaults to `2.0`." }, { "name": "offline", "kind": "field", "annotation": "bool", "default": "False", "required": false, "description": "Optional constructor field; defaults to `False`." } ], "return_annotation": "DownloadConfig", "docstring": "Configuration for model download behavior.", "summary": "Configuration for model download behavior.", "implementation": "Class `DownloadConfig` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 51, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/download.py#L45-L51", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "ensure_model_downloaded", "qualname": "ensure_model_downloaded", "full_name": "vllm_mlx.utils.download.ensure_model_downloaded", "kind": "function", "signature": "def ensure_model_downloaded(model_name: str, config: DownloadConfig | None=None, is_mllm: bool=False) -> Path", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path." }, { "name": "config", "kind": "positional or keyword", "annotation": "DownloadConfig | None", "default": "None", "required": false, "description": "Download configuration. Uses defaults if None." }, { "name": "is_mllm", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "If True, use MLLM download patterns (broader file set)." } ], "return_annotation": "Path", "docstring": "Ensure a model is available locally, downloading with retry if needed.\n\nArgs:\n model_name: HuggingFace model name or local path.\n config: Download configuration. Uses defaults if None.\n is_mllm: If True, use MLLM download patterns (broader file set).\n\nReturns:\n Path to the local model directory.\n\nRaises:\n RuntimeError: If download fails after all retries.\n KeyboardInterrupt: Propagated immediately without retry.", "summary": "Ensure a model is available locally, downloading with retry if needed.", "implementation": "Function `ensure_model_downloaded` calls `DownloadConfig`, `Path`, `model_path.exists`, `logger.info`; can raise `RuntimeError`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 54, "end_line": 144, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/download.py#L54-L144", "decorators": [], "calls": [ "DownloadConfig", "Path", "model_path.exists", "logger.info", "snapshot_download", "RuntimeError", "os.environ.get", "str", "range", "logger.warning", "time.sleep", "logger.error", "os.environ.pop" ], "state_reads": [], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "model_path", "result" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.utils.harmony_render", "path": "vllm_mlx/utils/harmony_render.py", "page_path": "reference/api/vllm_mlx/utils/harmony_render.md", "docstring": "Harmony-format prompt rendering for GPT-OSS via ``openai-harmony``.\n\nGPT-OSS models are trained with OpenAI's harmony wire format (channeled\n``<|start|>assistant<|channel|>commentary ...<|call|>`` tool calls,\n``<|start|>functions.X to=assistant<|channel|>commentary<|message|>...``\ntool results, etc.). Rendering harmony correctly from OpenAI-style chat\nmessages is delicate: prior assistant ``tool_calls`` must arrive at the\ntemplate as structural objects, not the bracket-text fallback that\n``api.utils.extract_multimodal_content()`` produces for non-native parsers.\n\nThis module bypasses the Jinja chat template entirely for harmony-active\nengines: it converts the OpenAI-format ``messages`` (plus ``tools``) to an\n``openai_harmony.Conversation`` and asks the library — the canonical\nrenderer maintained by OpenAI — to serialize it. That sidesteps both the\ntext-flattening upstream and any template-vs-training-format drift.\n\nThe library is an optional dependency. ``HAS_HARMONY`` reflects import\nsuccess so the rest of the engine can fall back to ``apply_chat_template``\nwhen the package is absent.\n\nSee https://github.com/waybarrios/vllm-mlx/issues/568 for the original\nreport and the patch shape Thump604 outlined.", "summary": "Harmony-format prompt rendering for GPT-OSS via ``openai-harmony``.", "line_count": 303, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L1-L303", "members": [ "logger", "_harmony_encoding", "is_harmony_parser_name", "_build_tools", "_content_to_text", "_convert_message", "_resolve_tool_names", "render_messages" ], "symbols": [ { "name": "_harmony_encoding", "qualname": "_harmony_encoding", "full_name": "vllm_mlx.utils.harmony_render._harmony_encoding", "kind": "function", "signature": "def _harmony_encoding() -> Any", "parameters": [], "return_annotation": "Any", "docstring": "Load the harmony encoding once and reuse it across requests.\n\n``load_harmony_encoding`` reads the harmony tokenizer assets, so calling\nit on every ``render_messages`` invocation would add latency to the\nper-request prompt build. Callers reach this only after the HAS_HARMONY\nguard in ``render_messages``, so ``_oh`` is never None here.", "summary": "Load the harmony encoding once and reuse it across requests.", "implementation": "Function `_harmony_encoding` calls `_oh.load_harmony_encoding`; returns `_oh.load_harmony_encoding(_oh.HarmonyEncodingName.HARMONY_GPT_OSS)`.", "documented": true, "public": false, "addressable": true, "line": 45, "end_line": 53, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L45-L53", "decorators": [ "lru_cache(maxsize=1)" ], "calls": [ "_oh.load_harmony_encoding" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_oh.load_harmony_encoding(_oh.HarmonyEncodingName.HARMONY_GPT_OSS)" ], "awaits": false, "yields": false }, { "name": "is_harmony_parser_name", "qualname": "is_harmony_parser_name", "full_name": "vllm_mlx.utils.harmony_render.is_harmony_parser_name", "kind": "function", "signature": "def is_harmony_parser_name(parser_name: str | None) -> bool", "parameters": [ { "name": "parser_name", "kind": "positional or keyword", "annotation": "str | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Return True when the active --tool-call-parser is a harmony alias.\n\n``HarmonyToolParser`` registers under both ``\"harmony\"`` and ``\"gpt-oss\"``.", "summary": "Return True when the active --tool-call-parser is a harmony alias.", "implementation": "Function `is_harmony_parser_name` returns `parser_name in {'harmony', 'gpt-oss'}`.", "documented": true, "public": true, "addressable": true, "line": 56, "end_line": 61, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L56-L61", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "parser_name in {'harmony', 'gpt-oss'}" ], "awaits": false, "yields": false }, { "name": "_build_tools", "qualname": "_build_tools", "full_name": "vllm_mlx.utils.harmony_render._build_tools", "kind": "function", "signature": "def _build_tools(tools: list[dict] | None) -> list[Any] | None", "parameters": [ { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[Any] | None", "docstring": "", "summary": "Function `_build_tools` calls `t.get`, `fn.get`, `tool_descs.append`, `_oh.ToolDescription.new`; has 2 explicit return paths.", "implementation": "Function `_build_tools` calls `t.get`, `fn.get`, `tool_descs.append`, `_oh.ToolDescription.new`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 64, "end_line": 80, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L64-L80", "decorators": [], "calls": [ "t.get", "fn.get", "tool_descs.append", "_oh.ToolDescription.new" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "tool_descs or None" ], "awaits": false, "yields": false }, { "name": "_content_to_text", "qualname": "_content_to_text", "full_name": "vllm_mlx.utils.harmony_render._content_to_text", "kind": "function", "signature": "def _content_to_text(content: Any) -> str", "parameters": [ { "name": "content", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Flatten OpenAI content (str | list[dict]) to plain text.", "summary": "Flatten OpenAI content (str | list[dict]) to plain text.", "implementation": "Function `_content_to_text` calls `isinstance`, `item.get`, `parts.append`, `'\\n'.join`; has 4 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 83, "end_line": 97, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L83-L97", "decorators": [], "calls": [ "isinstance", "item.get", "parts.append", "'\\n'.join", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "''", "content", "'\\n'.join(parts)", "str(content)" ], "awaits": false, "yields": false }, { "name": "_convert_message", "qualname": "_convert_message", "full_name": "vllm_mlx.utils.harmony_render._convert_message", "kind": "function", "signature": "def _convert_message(msg: dict) -> list[Any]", "parameters": [ { "name": "msg", "kind": "positional or keyword", "annotation": "dict", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[Any]", "docstring": "Convert one OpenAI-format message to one or more ``openai_harmony.Message``.\n\nA single assistant turn can carry multiple tool_calls; harmony represents\neach as its own commentary-channel message addressed to ``functions.X``.\nPrior reasoning lives in an analysis-channel message that precedes the\ntool calls.", "summary": "Convert one OpenAI-format message to one or more ``openai_harmony.Message``.", "implementation": "Function `_convert_message` calls `msg.get`, `_content_to_text`, `out.append`, `_oh.Message.from_role_and_content`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 100, "end_line": 182, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L100-L182", "decorators": [], "calls": [ "msg.get", "_content_to_text", "out.append", "_oh.Message.from_role_and_content", "tool_name.startswith", "_oh.Message", "_oh.Author.new", "_oh.TextContent", "str", "tc.get", "fn.get", "isinstance", "json.dumps" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[]", "out" ], "awaits": false, "yields": false }, { "name": "_resolve_tool_names", "qualname": "_resolve_tool_names", "full_name": "vllm_mlx.utils.harmony_render._resolve_tool_names", "kind": "function", "signature": "def _resolve_tool_names(messages: list[dict]) -> list[dict]", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "list[dict]", "docstring": "Stamp ``name=functions.X`` on each ``role=tool`` message by tracing back\nthe most recent assistant ``tool_call_id`` -> function name.", "summary": "Stamp ``name=functions.X`` on each ``role=tool`` message by tracing back the most recent assistant ``tool_call_id`` -> function name.", "implementation": "Function `_resolve_tool_names` calls `isinstance`, `out.append`, `m.get`, `tc.get`; returns `out`.", "documented": true, "public": false, "addressable": true, "line": 185, "end_line": 214, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L185-L214", "decorators": [], "calls": [ "isinstance", "out.append", "m.get", "tc.get", "fn.get", "dict", "new_m.get", "by_call_id.get" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "out" ], "awaits": false, "yields": false }, { "name": "render_messages", "qualname": "render_messages", "full_name": "vllm_mlx.utils.harmony_render.render_messages", "kind": "function", "signature": "def render_messages(messages: list[dict], tools: list[dict] | None=None, reasoning_effort: str | None=None) -> str", "parameters": [ { "name": "messages", "kind": "positional or keyword", "annotation": "list[dict]", "default": "", "required": true, "description": "OpenAI chat-completions messages." }, { "name": "tools", "kind": "positional or keyword", "annotation": "list[dict] | None", "default": "None", "required": false, "description": "OpenAI-format tools list (each item ``{\"type\":\"function\",\"function\":{...}}``)." }, { "name": "reasoning_effort", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "``\"low\"``, ``\"medium\"``, or ``\"high\"``. Defaults to medium." } ], "return_annotation": "str", "docstring": "Render OpenAI-format messages as a harmony-format prompt string.\n\nRaises ``RuntimeError`` if ``openai-harmony`` is not importable; callers\nshould pre-check with :data:`HAS_HARMONY` and fall back to\n``tokenizer.apply_chat_template`` when False.\n\nArgs:\n messages: OpenAI chat-completions messages.\n tools: OpenAI-format tools list (each item ``{\"type\":\"function\",\"function\":{...}}``).\n reasoning_effort: ``\"low\"``, ``\"medium\"``, or ``\"high\"``. Defaults to medium.\n\nReturns:\n Decoded harmony prompt with the trailing ``<|start|>assistant``\n marker ready for the model to begin generation.", "summary": "Render OpenAI-format messages as a harmony-format prompt string.", "implementation": "Function `render_messages` calls `RuntimeError`, `_resolve_tool_names`, `isinstance`, `other_msgs.append`; can raise `RuntimeError`; returns `enc.decode(token_ids)`.", "documented": true, "public": true, "addressable": true, "line": 217, "end_line": 303, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/harmony_render.py#L217-L303", "decorators": [], "calls": [ "RuntimeError", "_resolve_tool_names", "isinstance", "other_msgs.append", "m.get", "system_msgs.append", "developer_msgs.append", "_build_tools", "h_messages.extend", "_convert_message", "_oh.SystemContent.new", "getattr", "reasoning_effort.upper", "sys_content.with_reasoning_effort", "h_messages.append", "_oh.Message.from_role_and_content", "_oh.DeveloperContent.new().with_function_tools", "_oh.DeveloperContent.new", "_oh.Conversation.from_messages", "_harmony_encoding", "enc.render_conversation_for_completion", "enc.decode" ], "state_reads": [], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "enc.decode(token_ids)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.utils.mamba_cache", "path": "vllm_mlx/utils/mamba_cache.py", "page_path": "reference/api/vllm_mlx/utils/mamba_cache.md", "docstring": "BatchMambaCache implementation for continuous batching with Mamba models.\n\nmlx-lm's BatchGenerator requires cache objects to have an `extract` method,\nbut MambaCache (which extends ArraysCache) doesn't have one. This module\nprovides a BatchMambaCache wrapper that adds batching support.", "summary": "BatchMambaCache implementation for continuous batching with Mamba models.", "line_count": 215, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L1-L215", "members": [ "logger", "BatchMambaCache", "patch_mlx_lm_for_mamba", "_patched", "ensure_mamba_support" ], "symbols": [ { "name": "BatchMambaCache", "qualname": "BatchMambaCache", "full_name": "vllm_mlx.utils.mamba_cache.BatchMambaCache", "kind": "class", "signature": "class BatchMambaCache(MambaCache)", "parameters": [ { "name": "left_padding", "kind": "positional or keyword", "annotation": "Optional[List[int]]", "default": "None", "required": false, "description": "Amount of left padding for each sequence in batch" }, { "name": "size", "kind": "positional or keyword", "annotation": "int", "default": "2", "required": false, "description": "Number of state arrays (default 2 for Mamba models)" } ], "return_annotation": "BatchMambaCache", "docstring": "Batch-aware MambaCache for continuous batching.\n\nThis extends MambaCache to support batch operations required by\nmlx-lm's BatchGenerator, specifically the `extract` method.", "summary": "Batch-aware MambaCache for continuous batching.", "implementation": "Class `BatchMambaCache` derives from `MambaCache` and declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 24, "end_line": 96, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L24-L96", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "BatchMambaCache.__init__", "full_name": "vllm_mlx.utils.mamba_cache.BatchMambaCache.__init__", "kind": "method", "signature": "def __init__(self, left_padding: Optional[List[int]]=None, size: int=2)", "parameters": [ { "name": "left_padding", "kind": "positional or keyword", "annotation": "Optional[List[int]]", "default": "None", "required": false, "description": "Amount of left padding for each sequence in batch" }, { "name": "size", "kind": "positional or keyword", "annotation": "int", "default": "2", "required": false, "description": "Number of state arrays (default 2 for Mamba models)" } ], "return_annotation": "not annotated", "docstring": "Initialize BatchMambaCache.\n\nArgs:\n left_padding: Amount of left padding for each sequence in batch\n size: Number of state arrays (default 2 for Mamba models)", "summary": "Initialize BatchMambaCache.", "implementation": "Method `BatchMambaCache.__init__` updates `self._batch_size`; calls `super().__init__`, `super`, `len`.", "documented": true, "public": false, "addressable": true, "line": 32, "end_line": 43, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L32-L43", "decorators": [], "calls": [ "super().__init__", "super", "len" ], "state_reads": [], "state_writes": [ "self._batch_size" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "extract", "qualname": "BatchMambaCache.extract", "full_name": "vllm_mlx.utils.mamba_cache.BatchMambaCache.extract", "kind": "method", "signature": "def extract(self, idx: int) -> MambaCache", "parameters": [ { "name": "idx", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Index of the sequence to extract" } ], "return_annotation": "MambaCache", "docstring": "Extract a single cache from the batch.\n\nArgs:\n idx: Index of the sequence to extract\n\nReturns:\n A new MambaCache with the extracted state", "summary": "Extract a single cache from the batch.", "implementation": "Method `BatchMambaCache.extract` calls `len`, `MambaCache`, `mx.contiguous`; returns `cache`.", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L45-L63", "decorators": [], "calls": [ "len", "MambaCache", "mx.contiguous" ], "state_reads": [ "self.cache" ], "state_writes": [], "raises": [], "return_expressions": [ "cache" ], "awaits": false, "yields": false }, { "name": "merge", "qualname": "BatchMambaCache.merge", "full_name": "vllm_mlx.utils.mamba_cache.BatchMambaCache.merge", "kind": "method", "signature": "def merge(cls, caches: List[MambaCache]) -> 'BatchMambaCache'", "parameters": [ { "name": "caches", "kind": "positional or keyword", "annotation": "List[MambaCache]", "default": "", "required": true, "description": "List of MambaCache objects to merge" } ], "return_annotation": "'BatchMambaCache'", "docstring": "Merge multiple MambaCache objects into a BatchMambaCache.\n\nArgs:\n caches: List of MambaCache objects to merge\n\nReturns:\n A new BatchMambaCache containing all caches", "summary": "Merge multiple MambaCache objects into a BatchMambaCache.", "implementation": "Method `BatchMambaCache.merge` calls `cls`, `len`, `range`, `merged_cache.cache.append`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 66, "end_line": 96, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L66-L96", "decorators": [ "classmethod" ], "calls": [ "cls", "len", "range", "merged_cache.cache.append", "mx.concatenate" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "cls([])", "merged_cache" ], "awaits": false, "yields": false }, { "name": "patch_mlx_lm_for_mamba", "qualname": "patch_mlx_lm_for_mamba", "full_name": "vllm_mlx.utils.mamba_cache.patch_mlx_lm_for_mamba", "kind": "function", "signature": "def patch_mlx_lm_for_mamba()", "parameters": [], "return_annotation": "not annotated", "docstring": "Patch mlx-lm to support MambaCache in BatchGenerator.\n\nThis modifies the _make_cache function to handle MambaCache by\nconverting it to BatchMambaCache.", "summary": "Patch mlx-lm to support MambaCache in BatchGenerator.", "implementation": "Function `patch_mlx_lm_for_mamba` calls `importlib.import_module`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 99, "end_line": 194, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L99-L194", "decorators": [], "calls": [ "importlib.import_module", "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_patched_make_cache", "qualname": "patch_mlx_lm_for_mamba._patched_make_cache", "full_name": "vllm_mlx.utils.mamba_cache.patch_mlx_lm_for_mamba._patched_make_cache", "kind": "nested function", "signature": "def _patched_make_cache(model, left_padding, max_kv_size=None)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "The model to create cache for" }, { "name": "left_padding", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Left padding for batch" }, { "name": "max_kv_size", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Maximum KV cache size (mlx-lm 0.30.6+)" } ], "return_annotation": "not annotated", "docstring": "Convert a list of regular caches into their corresponding\nbatch-aware caches, with support for MambaCache.\n\nArgs:\n model: The model to create cache for\n left_padding: Left padding for batch\n max_kv_size: Maximum KV cache size (mlx-lm 0.30.6+)", "summary": "Convert a list of regular caches into their corresponding batch-aware caches, with support for MambaCache.", "implementation": "Nested Function `patch_mlx_lm_for_mamba._patched_make_cache` calls `hasattr`, `model.make_cache`, `to_batch_cache`, `BatchRotatingKVCache`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": false, "line": 126, "end_line": 166, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L126-L166", "decorators": [], "calls": [ "hasattr", "model.make_cache", "to_batch_cache", "BatchRotatingKVCache", "BatchKVCache" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[to_batch_cache(c) for c in cache]", "[BatchRotatingKVCache(max_kv_size, left_padding) for _ in model.layers]", "[BatchKVCache(left_padding) for _ in model.layers]" ], "awaits": false, "yields": false }, { "name": "to_batch_cache", "qualname": "patch_mlx_lm_for_mamba._patched_make_cache.to_batch_cache", "full_name": "vllm_mlx.utils.mamba_cache.patch_mlx_lm_for_mamba._patched_make_cache.to_batch_cache", "kind": "nested function", "signature": "def to_batch_cache(c)", "parameters": [ { "name": "c", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "", "summary": "Nested Function `patch_mlx_lm_for_mamba._patched_make_cache.to_batch_cache` calls `isinstance`, `BatchKVCache`, `BatchMambaCache`, `mx.array`; can raise `ValueError`; has 5 explicit return paths.", "implementation": "Nested Function `patch_mlx_lm_for_mamba._patched_make_cache.to_batch_cache` calls `isinstance`, `BatchKVCache`, `BatchMambaCache`, `mx.array`; can raise `ValueError`; has 5 explicit return paths.", "documented": false, "public": false, "addressable": false, "line": 137, "end_line": 155, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L137-L155", "decorators": [], "calls": [ "isinstance", "BatchKVCache", "BatchMambaCache", "mx.array", "ValueError", "BatchRotatingKVCache", "CacheList", "to_batch_cache", "type" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "BatchKVCache(left_padding)", "BatchMambaCache(left_padding)", "c", "BatchRotatingKVCache(c.max_size, left_padding)", "CacheList(*(to_batch_cache(sub_c) for sub_c in c.caches))" ], "awaits": false, "yields": false }, { "name": "_patched_merge_caches", "qualname": "patch_mlx_lm_for_mamba._patched_merge_caches", "full_name": "vllm_mlx.utils.mamba_cache.patch_mlx_lm_for_mamba._patched_merge_caches", "kind": "nested function", "signature": "def _patched_merge_caches(caches)", "parameters": [ { "name": "caches", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Merge caches with MambaCache support.", "summary": "Merge caches with MambaCache support.", "implementation": "Nested Function `patch_mlx_lm_for_mamba._patched_merge_caches` calls `range`, `len`, `isinstance`, `BatchKVCache.merge`; can raise `ValueError`; returns `batch_cache`.", "documented": true, "public": false, "addressable": false, "line": 174, "end_line": 190, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L174-L190", "decorators": [], "calls": [ "range", "len", "isinstance", "BatchKVCache.merge", "BatchRotatingKVCache.merge", "BatchMambaCache.merge", "ValueError", "type", "batch_cache.append" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "batch_cache" ], "awaits": false, "yields": false }, { "name": "ensure_mamba_support", "qualname": "ensure_mamba_support", "full_name": "vllm_mlx.utils.mamba_cache.ensure_mamba_support", "kind": "function", "signature": "def ensure_mamba_support()", "parameters": [], "return_annotation": "not annotated", "docstring": "Ensure MambaCache batching support is enabled.\n\nNOTE: Disabled for mlx-lm >= 0.30.6 where ArraysCache natively supports\nall batch operations (extract, merge, filter, prepare). The old patch\nreplaced ArraysCache with BatchMambaCache, which broke hybrid models\n(Qwen3.5) that mix ArraysCache + KVCache layers.", "summary": "Ensure MambaCache batching support is enabled.", "implementation": "Function `ensure_mamba_support` calls `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 201, "end_line": 215, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/mamba_cache.py#L201-L215", "decorators": [], "calls": [ "logger.info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.utils.tokenizer", "path": "vllm_mlx/utils/tokenizer.py", "page_path": "reference/api/vllm_mlx/utils/tokenizer.md", "docstring": "Tokenizer utilities with fallback support for non-standard tokenizers.\n\nSome models (e.g., Nemotron) use non-standard tokenizer configurations\nthat transformers doesn't recognize. This module provides fallback loading\ndirectly from tokenizer.json.", "summary": "Tokenizer utilities with fallback support for non-standard tokenizers.", "line_count": 280, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L1-L280", "members": [ "logger", "FALLBACK_MODELS", "_needs_tokenizer_fallback", "_needs_strict_false", "load_model_with_fallback", "_load_strict_false", "_try_inject_mtp", "_try_inject_mtp_post_load", "_load_with_tokenizer_fallback" ], "symbols": [ { "name": "_needs_tokenizer_fallback", "qualname": "_needs_tokenizer_fallback", "full_name": "vllm_mlx.utils.tokenizer._needs_tokenizer_fallback", "kind": "function", "signature": "def _needs_tokenizer_fallback(model_name: str) -> bool", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if model needs tokenizer fallback.", "summary": "Check if model needs tokenizer fallback.", "implementation": "Function `_needs_tokenizer_fallback` calls `model_name.lower`, `any`, `pattern.lower`; returns `any((pattern.lower() in model_lower for pattern in FALLBACK_MODELS))`.", "documented": true, "public": false, "addressable": true, "line": 25, "end_line": 28, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L25-L28", "decorators": [], "calls": [ "model_name.lower", "any", "pattern.lower" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "any((pattern.lower() in model_lower for pattern in FALLBACK_MODELS))" ], "awaits": false, "yields": false }, { "name": "_needs_strict_false", "qualname": "_needs_strict_false", "full_name": "vllm_mlx.utils.tokenizer._needs_strict_false", "kind": "function", "signature": "def _needs_strict_false(model_name: str) -> bool", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Check if model needs strict=False loading (VLM models with extra weights).\n\nVLM models (e.g., Qwen3.5) have vision_tower weights that don't match\nthe text-only model class. Loading with strict=True fails and wastes\nmemory by loading all weights (~100 GB) before raising ValueError.\nDetect these models up-front to avoid the double-load penalty.", "summary": "Check if model needs strict=False loading (VLM models with extra weights).", "implementation": "Function `_needs_strict_false` calls `_download`, `load_config`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 31, "end_line": 49, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L31-L49", "decorators": [], "calls": [ "_download", "load_config" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False", "True" ], "awaits": false, "yields": false }, { "name": "load_model_with_fallback", "qualname": "load_model_with_fallback", "full_name": "vllm_mlx.utils.tokenizer.load_model_with_fallback", "kind": "function", "signature": "def load_model_with_fallback(model_name: str, tokenizer_config: dict=None)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "HuggingFace model name or local path" }, { "name": "tokenizer_config", "kind": "positional or keyword", "annotation": "dict", "default": "None", "required": false, "description": "Optional tokenizer configuration" } ], "return_annotation": "not annotated", "docstring": "Load model and tokenizer with fallback for non-standard tokenizers.\n\nArgs:\n model_name: HuggingFace model name or local path\n tokenizer_config: Optional tokenizer configuration\n\nReturns:\n Tuple of (model, tokenizer)", "summary": "Load model and tokenizer with fallback for non-standard tokenizers.", "implementation": "Function `load_model_with_fallback` calls `_needs_tokenizer_fallback`, `logger.info`, `_load_with_tokenizer_fallback`, `_needs_strict_false`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 52, "end_line": 111, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L52-L111", "decorators": [], "calls": [ "_needs_tokenizer_fallback", "logger.info", "_load_with_tokenizer_fallback", "_needs_strict_false", "_load_strict_false", "load", "str", "logger.warning", "gc.collect", "_try_inject_mtp_post_load" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_load_with_tokenizer_fallback(model_name)", "_load_strict_false(model_name, tokenizer_config)", "(model, tokenizer)" ], "awaits": false, "yields": false }, { "name": "_load_strict_false", "qualname": "_load_strict_false", "full_name": "vllm_mlx.utils.tokenizer._load_strict_false", "kind": "function", "signature": "def _load_strict_false(model_name: str, tokenizer_config: dict=None)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "tokenizer_config", "kind": "positional or keyword", "annotation": "dict", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "not annotated", "docstring": "Load model with strict=False to discard extra weights.\n\nHandles models with extra parameters that the text-only model class\ndoesn't define (e.g., vision tower weights in VLM models like Qwen3.5,\nor MTP layers). The model's own sanitize() handles key remapping\n(e.g., language_model.* prefix), and strict=False silently drops\nunmatched keys.", "summary": "Load model with strict=False to discard extra weights.", "implementation": "Function `_load_strict_false` calls `_download`, `load_model`, `tree_flatten`, `model.parameters`; returns `(model, tokenizer)`.", "documented": true, "public": false, "addressable": true, "line": 114, "end_line": 153, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L114-L153", "decorators": [], "calls": [ "_download", "load_model", "tree_flatten", "model.parameters", "len", "sum", "mx.all(v == 0).item", "mx.all", "logger.info", "hasattr", "mx.mean(emb.astype(mx.float32)).item", "mx.mean", "emb.astype", "load_tokenizer", "config.get", "_try_inject_mtp" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "(model, tokenizer)" ], "awaits": false, "yields": false }, { "name": "_try_inject_mtp", "qualname": "_try_inject_mtp", "full_name": "vllm_mlx.utils.tokenizer._try_inject_mtp", "kind": "function", "signature": "def _try_inject_mtp(model, model_path, config)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model_path", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "config", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Inject MTP support if model has MTP config + weights.", "summary": "Inject MTP support if model has MTP config + weights.", "implementation": "Function `_try_inject_mtp` calls `config.get`, `text_config.get`, `inject_mtp_support`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 156, "end_line": 176, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L156-L176", "decorators": [], "calls": [ "config.get", "text_config.get", "inject_mtp_support" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_try_inject_mtp_post_load", "qualname": "_try_inject_mtp_post_load", "full_name": "vllm_mlx.utils.tokenizer._try_inject_mtp_post_load", "kind": "function", "signature": "def _try_inject_mtp_post_load(model, model_name)", "parameters": [ { "name": "model", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "model_name", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Check if MTP weights exist but were stripped by sanitize(), and inject.", "summary": "Check if MTP weights exist but were stripped by sanitize(), and inject.", "implementation": "Function `_try_inject_mtp_post_load` calls `_download`, `Path`, `config_path.exists`, `open`; returns `None`.", "documented": true, "public": false, "addressable": true, "line": 179, "end_line": 215, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L179-L215", "decorators": [], "calls": [ "_download", "Path", "config_path.exists", "open", "json.load", "config.get", "text_config.get", "hasattr", "getattr", "mtp_file.exists", "logger.info", "_try_inject_mtp" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "_load_with_tokenizer_fallback", "qualname": "_load_with_tokenizer_fallback", "full_name": "vllm_mlx.utils.tokenizer._load_with_tokenizer_fallback", "kind": "function", "signature": "def _load_with_tokenizer_fallback(model_name: str)", "parameters": [ { "name": "model_name", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "not annotated", "docstring": "Load model with fallback tokenizer for non-standard models like Nemotron.", "summary": "Load model with fallback tokenizer for non-standard models like Nemotron.", "implementation": "Function `_load_with_tokenizer_fallback` calls `logger.info`, `ensure_model_downloaded`, `load_model`, `tokenizer_json.exists`; can raise `ValueError`; returns `(model, tokenizer)`.", "documented": true, "public": false, "addressable": true, "line": 218, "end_line": 280, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/tokenizer.py#L218-L280", "decorators": [], "calls": [ "logger.info", "ensure_model_downloaded", "load_model", "tokenizer_json.exists", "Tokenizer.from_file", "str", "tokenizer_config_path.exists", "open", "json.load", "config.get", "PreTrainedTokenizerFast", "_needs_tokenizer_fallback", "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [ "(model, tokenizer)" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.utils.truncation", "path": "vllm_mlx/utils/truncation.py", "page_path": "reference/api/vllm_mlx/utils/truncation.md", "docstring": "Shared resolution of the tokenizer truncation length for embedding and\nreranker models.\n\nThe input token limit follows each model's own context window\n(``max_position_embeddings``) instead of a hard-coded 512. A finite tokenizer\nlimit further constrains that architecture value, while HuggingFace's huge\nunset placeholder is ignored. This keeps position-table offsets safe for\nRoBERTa-family models without losing model-derived limits for sentinel values.", "summary": "Shared resolution of the tokenizer truncation length for embedding and reranker models.", "line_count": 83, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/truncation.py#L1-L83", "members": [ "MAX_LENGTH_DEFAULT", "TOKENIZER_SENTINEL_THRESHOLD", "_config_get", "inner_tokenizer", "_positive_int", "resolve_max_length" ], "symbols": [ { "name": "_config_get", "qualname": "_config_get", "full_name": "vllm_mlx.utils.truncation._config_get", "kind": "function", "signature": "def _config_get(config: Any, key: str) -> Any", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "key", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Read ``key`` from a model config that may be a dict or an object.", "summary": "Read ``key`` from a model config that may be a dict or an object.", "implementation": "Function `_config_get` calls `isinstance`, `config.get`, `getattr`; has 3 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 25, "end_line": 31, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/truncation.py#L25-L31", "decorators": [], "calls": [ "isinstance", "config.get", "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "None", "config.get(key)", "getattr(config, key, None)" ], "awaits": false, "yields": false }, { "name": "inner_tokenizer", "qualname": "inner_tokenizer", "full_name": "vllm_mlx.utils.truncation.inner_tokenizer", "kind": "function", "signature": "def inner_tokenizer(tokenizer: Any) -> Any", "parameters": [ { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Any", "docstring": "Unwrap a wrapping tokenizer to its inner ``_tokenizer`` when present.", "summary": "Unwrap a wrapping tokenizer to its inner ``_tokenizer`` when present.", "implementation": "Function `inner_tokenizer` calls `getattr`; returns `getattr(tokenizer, '_tokenizer', tokenizer)`.", "documented": true, "public": true, "addressable": true, "line": 34, "end_line": 36, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/truncation.py#L34-L36", "decorators": [], "calls": [ "getattr" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "getattr(tokenizer, '_tokenizer', tokenizer)" ], "awaits": false, "yields": false }, { "name": "_positive_int", "qualname": "_positive_int", "full_name": "vllm_mlx.utils.truncation._positive_int", "kind": "function", "signature": "def _positive_int(value: Any) -> int | None", "parameters": [ { "name": "value", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "int | None", "docstring": "", "summary": "Function `_positive_int` calls `isinstance`; has 2 explicit return paths.", "implementation": "Function `_positive_int` calls `isinstance`; has 2 explicit return paths.", "documented": false, "public": false, "addressable": true, "line": 39, "end_line": 42, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/truncation.py#L39-L42", "decorators": [], "calls": [ "isinstance" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "value", "None" ], "awaits": false, "yields": false }, { "name": "resolve_max_length", "qualname": "resolve_max_length", "full_name": "vllm_mlx.utils.truncation.resolve_max_length", "kind": "function", "signature": "def resolve_max_length(config: Any, tokenizer: Any, *, default: int=MAX_LENGTH_DEFAULT, sentinel_threshold: int=TOKENIZER_SENTINEL_THRESHOLD) -> int", "parameters": [ { "name": "config", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "Model config as a dict (reranker) or object (embeddings)." }, { "name": "tokenizer", "kind": "positional or keyword", "annotation": "Any", "default": "", "required": true, "description": "The tokenizer (possibly wrapping an inner ``_tokenizer``)." }, { "name": "default", "kind": "keyword-only", "annotation": "int", "default": "MAX_LENGTH_DEFAULT", "required": false, "description": "Fallback when no usable value is found." }, { "name": "sentinel_threshold", "kind": "keyword-only", "annotation": "int", "default": "TOKENIZER_SENTINEL_THRESHOLD", "required": false, "description": "Tokenizer-derived values at or above this are treated as an unset HuggingFace sentinel, not a real length." } ], "return_annotation": "int", "docstring": "Resolve the tokenizer truncation length for a model.\n\nSource order:\n 1. ``config.max_position_embeddings`` — an explicit value supplied by\n the model's own architecture.\n 2. A finite ``tokenizer.model_max_length`` further constrains the\n architecture value. This matters for RoBERTa-family models, whose\n position table includes reserved padding positions.\n 3. ``default``, when neither source yields a usable value.\n\nArgs:\n config: Model config as a dict (reranker) or object (embeddings).\n tokenizer: The tokenizer (possibly wrapping an inner ``_tokenizer``).\n default: Fallback when no usable value is found.\n sentinel_threshold: Tokenizer-derived values at or above this are\n treated as an unset HuggingFace sentinel, not a real length.\n\nReturns:\n The truncation length to pass as ``max_length``.", "summary": "Resolve the tokenizer truncation length for a model.", "implementation": "Function `resolve_max_length` calls `_positive_int`, `_config_get`, `getattr`, `inner_tokenizer`; returns `resolved`.", "documented": true, "public": true, "addressable": true, "line": 45, "end_line": 83, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/utils/truncation.py#L45-L83", "decorators": [], "calls": [ "_positive_int", "_config_get", "getattr", "inner_tokenizer", "min" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "resolved" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.vision_embedding_cache", "path": "vllm_mlx/vision_embedding_cache.py", "page_path": "reference/api/vllm_mlx/vision_embedding_cache.md", "docstring": "Vision Embedding Cache for MLLM continuous batching.\n\nThis module provides caching for vision embeddings to avoid redundant\ncomputation when the same images are processed multiple times.\n\nCache Levels:\n1. Pixel Values Cache - Caches processed image tensors (prepare_inputs output)\n2. Vision Encoding Cache - Caches VLM forward pass output (logits + cache state)\n\nPerformance Impact:\n- Without cache: ~2s per image for vision encoding\n- With cache hit: ~0.01s (100x speedup for repeated images)", "summary": "Vision Embedding Cache for MLLM continuous batching.", "line_count": 413, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L1-L413", "members": [ "logger", "VisionCacheStats", "PixelCacheEntry", "PixelOnlyCacheEntry", "EncodingCacheEntry", "compute_image_hash", "compute_images_hash", "VisionEmbeddingCache" ], "symbols": [ { "name": "VisionCacheStats", "qualname": "VisionCacheStats", "full_name": "vllm_mlx.vision_embedding_cache.VisionCacheStats", "kind": "class", "signature": "class VisionCacheStats", "parameters": [ { "name": "pixel_cache_hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "pixel_cache_misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "encoding_cache_hits", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "encoding_cache_misses", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." }, { "name": "total_time_saved", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." }, { "name": "total_images_processed", "kind": "field", "annotation": "int", "default": "0", "required": false, "description": "Optional constructor field; defaults to `0`." } ], "return_annotation": "VisionCacheStats", "docstring": "Statistics for vision cache performance.", "summary": "Statistics for vision cache performance.", "implementation": "Class `VisionCacheStats` declares 3 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 30, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L30-L66", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "pixel_hit_rate", "qualname": "VisionCacheStats.pixel_hit_rate", "full_name": "vllm_mlx.vision_embedding_cache.VisionCacheStats.pixel_hit_rate", "kind": "method", "signature": "def pixel_hit_rate(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Return successful pixel-cache lookups divided by all pixel lookups.", "summary": "Return successful pixel-cache lookups divided by all pixel lookups.", "implementation": "Method `VisionCacheStats.pixel_hit_rate` returns `self.pixel_cache_hits / total if total > 0 else 0.0`.", "documented": true, "public": true, "addressable": true, "line": 41, "end_line": 45, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L41-L45", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.pixel_cache_hits", "self.pixel_cache_misses" ], "state_writes": [], "raises": [], "return_expressions": [ "self.pixel_cache_hits / total if total > 0 else 0.0" ], "awaits": false, "yields": false }, { "name": "encoding_hit_rate", "qualname": "VisionCacheStats.encoding_hit_rate", "full_name": "vllm_mlx.vision_embedding_cache.VisionCacheStats.encoding_hit_rate", "kind": "method", "signature": "def encoding_hit_rate(self) -> float", "parameters": [], "return_annotation": "float", "docstring": "Return successful encoding lookups divided by all encoding lookups.", "summary": "Return successful encoding lookups divided by all encoding lookups.", "implementation": "Method `VisionCacheStats.encoding_hit_rate` returns `self.encoding_cache_hits / total if total > 0 else 0.0`.", "documented": true, "public": true, "addressable": true, "line": 48, "end_line": 52, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L48-L52", "decorators": [ "property" ], "calls": [], "state_reads": [ "self.encoding_cache_hits", "self.encoding_cache_misses" ], "state_writes": [], "raises": [], "return_expressions": [ "self.encoding_cache_hits / total if total > 0 else 0.0" ], "awaits": false, "yields": false }, { "name": "to_dict", "qualname": "VisionCacheStats.to_dict", "full_name": "vllm_mlx.vision_embedding_cache.VisionCacheStats.to_dict", "kind": "method", "signature": "def to_dict(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Return pixel, encoding, timing, and image counters.", "summary": "Return pixel, encoding, timing, and image counters.", "implementation": "Method `VisionCacheStats.to_dict` returns `{'pixel_cache_hits': self.pixel_cache_hits, 'pixel_cache_misses': self.pixel_cache_misses, 'pixel_hit_rate': self.pixel…`.", "documented": true, "public": true, "addressable": true, "line": 54, "end_line": 66, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L54-L66", "decorators": [], "calls": [], "state_reads": [ "self.pixel_cache_hits", "self.pixel_cache_misses", "self.pixel_hit_rate", "self.encoding_cache_hits", "self.encoding_cache_misses", "self.encoding_hit_rate", "self.total_time_saved", "self.total_images_processed" ], "state_writes": [], "raises": [], "return_expressions": [ "{'pixel_cache_hits': self.pixel_cache_hits, 'pixel_cache_misses': self.pixel_cache_misses, 'pixel_hit_rate': self.pixel…" ], "awaits": false, "yields": false }, { "name": "PixelCacheEntry", "qualname": "PixelCacheEntry", "full_name": "vllm_mlx.vision_embedding_cache.PixelCacheEntry", "kind": "class", "signature": "class PixelCacheEntry", "parameters": [ { "name": "pixel_values", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "input_ids", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "attention_mask", "kind": "field", "annotation": "Optional[mx.array]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "image_grid_thw", "kind": "field", "annotation": "Optional[mx.array]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "extra_kwargs", "kind": "field", "annotation": "Dict[str, Any]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "processing_time", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "PixelCacheEntry", "docstring": "Cached pixel values from prepare_inputs.", "summary": "Cached pixel values from prepare_inputs.", "implementation": "Class `PixelCacheEntry` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 70, "end_line": 78, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L70-L78", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "PixelOnlyCacheEntry", "qualname": "PixelOnlyCacheEntry", "full_name": "vllm_mlx.vision_embedding_cache.PixelOnlyCacheEntry", "kind": "class", "signature": "class PixelOnlyCacheEntry", "parameters": [ { "name": "pixel_values", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "image_grid_thw", "kind": "field", "annotation": "Optional[mx.array]", "default": "", "required": true, "description": "Required constructor field." }, { "name": "processing_time", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "PixelOnlyCacheEntry", "docstring": "Cached pixel values only (prompt-independent).\n\nThis cache stores only the image-dependent data that doesn't\nchange with different prompts. Useful when the same images\nare used with different prompts.", "summary": "Cached pixel values only (prompt-independent).", "implementation": "Class `PixelOnlyCacheEntry` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 82, "end_line": 92, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L82-L92", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "EncodingCacheEntry", "qualname": "EncodingCacheEntry", "full_name": "vllm_mlx.vision_embedding_cache.EncodingCacheEntry", "kind": "class", "signature": "class EncodingCacheEntry", "parameters": [ { "name": "logits", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "first_token", "kind": "field", "annotation": "int", "default": "", "required": true, "description": "Required constructor field." }, { "name": "logprobs", "kind": "field", "annotation": "mx.array", "default": "", "required": true, "description": "Required constructor field." }, { "name": "encoding_time", "kind": "field", "annotation": "float", "default": "0.0", "required": false, "description": "Optional constructor field; defaults to `0.0`." } ], "return_annotation": "EncodingCacheEntry", "docstring": "Cached vision encoding output.", "summary": "Cached vision encoding output.", "implementation": "Class `EncodingCacheEntry` declares 0 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 96, "end_line": 102, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L96-L102", "decorators": [ "dataclass" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "compute_image_hash", "qualname": "compute_image_hash", "full_name": "vllm_mlx.vision_embedding_cache.compute_image_hash", "kind": "function", "signature": "def compute_image_hash(image_path: str) -> str", "parameters": [ { "name": "image_path", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Compute hash of image content.\n\nFor files: hash the actual content\nFor URLs/base64: hash the string", "summary": "Compute hash of image content.", "implementation": "Function `compute_image_hash` calls `Path`, `path.exists`, `path.is_file`, `open`; has 3 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 105, "end_line": 124, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L105-L124", "decorators": [], "calls": [ "Path", "path.exists", "path.is_file", "open", "f.read", "hashlib.sha256(content).hexdigest", "hashlib.sha256", "hashlib.sha256(image_path.encode()).hexdigest", "image_path.encode", "hashlib.sha256(str(image_path).encode()).hexdigest", "str(image_path).encode", "str" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "hashlib.sha256(content).hexdigest()[:16]", "hashlib.sha256(image_path.encode()).hexdigest()[:16]", "hashlib.sha256(str(image_path).encode()).hexdigest()[:16]" ], "awaits": false, "yields": false }, { "name": "compute_images_hash", "qualname": "compute_images_hash", "full_name": "vllm_mlx.vision_embedding_cache.compute_images_hash", "kind": "function", "signature": "def compute_images_hash(images: List[str]) -> str", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Compute combined hash for multiple images.", "summary": "Compute combined hash for multiple images.", "implementation": "Function `compute_images_hash` calls `sorted`, `compute_image_hash`, `hashlib.sha256('_'.join(hashes).encode()).hexdigest`, `hashlib.sha256`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 127, "end_line": 132, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L127-L132", "decorators": [], "calls": [ "sorted", "compute_image_hash", "hashlib.sha256('_'.join(hashes).encode()).hexdigest", "hashlib.sha256", "'_'.join(hashes).encode", "'_'.join" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'no_images'", "hashlib.sha256('_'.join(hashes).encode()).hexdigest()[:16]" ], "awaits": false, "yields": false }, { "name": "VisionEmbeddingCache", "qualname": "VisionEmbeddingCache", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache", "kind": "class", "signature": "class VisionEmbeddingCache", "parameters": [ { "name": "max_pixel_entries", "kind": "positional or keyword", "annotation": "int", "default": "100", "required": false, "description": "Max entries in pixel cache (LRU eviction)" }, { "name": "max_encoding_entries", "kind": "positional or keyword", "annotation": "int", "default": "50", "required": false, "description": "Max entries in encoding cache" }, { "name": "enabled", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether caching is enabled" } ], "return_annotation": "VisionEmbeddingCache", "docstring": "Two-level cache for vision processing in MLLM.\n\nLevel 1 (Pixel Cache):\n - Caches output of prepare_inputs() (pixel_values, input_ids, etc.)\n - Key: hash(images) + hash(prompt)\n - Saves: Image loading, resizing, normalization time (~0.5-1s)\n\nLevel 2 (Encoding Cache):\n - Caches output of VLM forward pass (logits, first token)\n - Key: hash(images) + hash(prompt)\n - Saves: Vision encoder computation time (~1-2s)\n\nExample:\n >>> cache = VisionEmbeddingCache(max_pixel_entries=50, max_encoding_entries=20)\n >>>\n >>> # First request - cache miss\n >>> pixel_entry = cache.get_pixel_cache(images, prompt)\n >>> if pixel_entry is None:\n ... # Process images...\n ... cache.set_pixel_cache(images, prompt, pixel_values, ...)\n >>>\n >>> # Second request with same image - cache hit!\n >>> pixel_entry = cache.get_pixel_cache(images, prompt) # Returns cached data", "summary": "Two-level cache for vision processing in MLLM.", "implementation": "Class `VisionEmbeddingCache` declares 12 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 135, "end_line": 413, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L135-L413", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "VisionEmbeddingCache.__init__", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.__init__", "kind": "method", "signature": "def __init__(self, max_pixel_entries: int=100, max_encoding_entries: int=50, enabled: bool=True)", "parameters": [ { "name": "max_pixel_entries", "kind": "positional or keyword", "annotation": "int", "default": "100", "required": false, "description": "Max entries in pixel cache (LRU eviction)" }, { "name": "max_encoding_entries", "kind": "positional or keyword", "annotation": "int", "default": "50", "required": false, "description": "Max entries in encoding cache" }, { "name": "enabled", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Whether caching is enabled" } ], "return_annotation": "not annotated", "docstring": "Initialize the vision embedding cache.\n\nArgs:\n max_pixel_entries: Max entries in pixel cache (LRU eviction)\n max_encoding_entries: Max entries in encoding cache\n enabled: Whether caching is enabled", "summary": "Initialize the vision embedding cache.", "implementation": "Method `VisionEmbeddingCache.__init__` updates `self.max_pixel_entries`, `self.max_encoding_entries`, `self.enabled`, `self._pixel_cache`; calls `OrderedDict`, `VisionCacheStats`.", "documented": true, "public": false, "addressable": true, "line": 162, "end_line": 185, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L162-L185", "decorators": [], "calls": [ "OrderedDict", "VisionCacheStats" ], "state_reads": [], "state_writes": [ "self.max_pixel_entries", "self.max_encoding_entries", "self.enabled", "self._pixel_cache", "self._pixel_only_cache", "self._encoding_cache", "self.stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_make_key", "qualname": "VisionEmbeddingCache._make_key", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache._make_key", "kind": "method", "signature": "def _make_key(self, images: List[str], prompt: str) -> str", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Create cache key from images and prompt.", "summary": "Create cache key from images and prompt.", "implementation": "Method `VisionEmbeddingCache._make_key` calls `compute_images_hash`, `hashlib.sha256(prompt.encode()).hexdigest`, `hashlib.sha256`, `prompt.encode`; returns `f'{img_hash}_{prompt_hash}'`.", "documented": true, "public": false, "addressable": true, "line": 187, "end_line": 192, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L187-L192", "decorators": [], "calls": [ "compute_images_hash", "hashlib.sha256(prompt.encode()).hexdigest", "hashlib.sha256", "prompt.encode" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "f'{img_hash}_{prompt_hash}'" ], "awaits": false, "yields": false }, { "name": "_make_image_only_key", "qualname": "VisionEmbeddingCache._make_image_only_key", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache._make_image_only_key", "kind": "method", "signature": "def _make_image_only_key(self, images: List[str]) -> str", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "str", "docstring": "Create cache key from images only (prompt-independent).", "summary": "Create cache key from images only (prompt-independent).", "implementation": "Method `VisionEmbeddingCache._make_image_only_key` calls `compute_images_hash`; returns `compute_images_hash(images)`.", "documented": true, "public": false, "addressable": true, "line": 194, "end_line": 196, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L194-L196", "decorators": [], "calls": [ "compute_images_hash" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "compute_images_hash(images)" ], "awaits": false, "yields": false }, { "name": "get_pixel_cache", "qualname": "VisionEmbeddingCache.get_pixel_cache", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.get_pixel_cache", "kind": "method", "signature": "def get_pixel_cache(self, images: List[str], prompt: str) -> Optional[PixelCacheEntry]", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[PixelCacheEntry]", "docstring": "Get cached pixel values for images+prompt.\n\nReturns:\n PixelCacheEntry if found, None otherwise", "summary": "Get cached pixel values for images+prompt.", "implementation": "Method `VisionEmbeddingCache.get_pixel_cache` updates `self.stats.pixel_cache_hits`, `self.stats.total_time_saved`, `self.stats.pixel_cache_misses`; calls `self._make_key`, `self._pixel_cache.pop`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 200, "end_line": 229, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L200-L229", "decorators": [], "calls": [ "self._make_key", "self._pixel_cache.pop", "logger.debug" ], "state_reads": [ "self.enabled", "self._make_key", "self._pixel_cache", "self._pixel_cache.pop", "self.stats" ], "state_writes": [ "self.stats.pixel_cache_hits", "self.stats.total_time_saved", "self.stats.pixel_cache_misses" ], "raises": [], "return_expressions": [ "None", "entry" ], "awaits": false, "yields": false }, { "name": "set_pixel_cache", "qualname": "VisionEmbeddingCache.set_pixel_cache", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.set_pixel_cache", "kind": "method", "signature": "def set_pixel_cache(self, images: List[str], prompt: str, pixel_values: mx.array, input_ids: mx.array, attention_mask: Optional[mx.array]=None, image_grid_thw: Optional[mx.array]=None, extra_kwargs: Optional[Dict[str, Any]]=None, processing_time: float=0.0) -> None", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "pixel_values", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "input_ids", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attention_mask", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "image_grid_thw", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "extra_kwargs", "kind": "positional or keyword", "annotation": "Optional[Dict[str, Any]]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "processing_time", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Optional positional or keyword input; defaults to `0.0`." } ], "return_annotation": "None", "docstring": "Store pixel values in cache.", "summary": "Store pixel values in cache.", "implementation": "Method `VisionEmbeddingCache.set_pixel_cache` updates `self.stats.total_images_processed`; calls `self._make_key`, `len`, `next`, `iter`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 231, "end_line": 264, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L231-L264", "decorators": [], "calls": [ "self._make_key", "len", "next", "iter", "logger.debug", "PixelCacheEntry" ], "state_reads": [ "self.enabled", "self._make_key", "self._pixel_cache", "self.max_pixel_entries", "self.stats" ], "state_writes": [ "self.stats.total_images_processed" ], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_pixel_values", "qualname": "VisionEmbeddingCache.get_pixel_values", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.get_pixel_values", "kind": "method", "signature": "def get_pixel_values(self, images: List[str]) -> Optional[PixelOnlyCacheEntry]", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[PixelOnlyCacheEntry]", "docstring": "Get cached pixel values for images (prompt-independent).\n\nThis is useful when the same images are used with different prompts.\nOnly the pixel_values and image_grid_thw are cached (no input_ids).\n\nReturns:\n PixelOnlyCacheEntry if found, None otherwise", "summary": "Get cached pixel values for images (prompt-independent).", "implementation": "Method `VisionEmbeddingCache.get_pixel_values` updates `self.stats.pixel_cache_hits`, `self.stats.total_time_saved`, `self.stats.pixel_cache_misses`; calls `self._make_image_only_key`, `self._pixel_only_cache.pop`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 268, "end_line": 299, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L268-L299", "decorators": [], "calls": [ "self._make_image_only_key", "self._pixel_only_cache.pop", "logger.debug" ], "state_reads": [ "self.enabled", "self._make_image_only_key", "self._pixel_only_cache", "self._pixel_only_cache.pop", "self.stats" ], "state_writes": [ "self.stats.pixel_cache_hits", "self.stats.total_time_saved", "self.stats.pixel_cache_misses" ], "raises": [], "return_expressions": [ "None", "entry" ], "awaits": false, "yields": false }, { "name": "set_pixel_values", "qualname": "VisionEmbeddingCache.set_pixel_values", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.set_pixel_values", "kind": "method", "signature": "def set_pixel_values(self, images: List[str], pixel_values: mx.array, image_grid_thw: Optional[mx.array]=None, processing_time: float=0.0) -> None", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "pixel_values", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "image_grid_thw", "kind": "positional or keyword", "annotation": "Optional[mx.array]", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." }, { "name": "processing_time", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Optional positional or keyword input; defaults to `0.0`." } ], "return_annotation": "None", "docstring": "Store pixel values in cache (prompt-independent).", "summary": "Store pixel values in cache (prompt-independent).", "implementation": "Method `VisionEmbeddingCache.set_pixel_values` calls `self._make_image_only_key`, `len`, `next`, `iter`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 301, "end_line": 326, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L301-L326", "decorators": [], "calls": [ "self._make_image_only_key", "len", "next", "iter", "logger.debug", "PixelOnlyCacheEntry" ], "state_reads": [ "self.enabled", "self._make_image_only_key", "self._pixel_only_cache", "self.max_pixel_entries" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_encoding_cache", "qualname": "VisionEmbeddingCache.get_encoding_cache", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.get_encoding_cache", "kind": "method", "signature": "def get_encoding_cache(self, images: List[str], prompt: str) -> Optional[EncodingCacheEntry]", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "Optional[EncodingCacheEntry]", "docstring": "Get cached vision encoding output.\n\nReturns:\n EncodingCacheEntry if found, None otherwise", "summary": "Get cached vision encoding output.", "implementation": "Method `VisionEmbeddingCache.get_encoding_cache` updates `self.stats.encoding_cache_hits`, `self.stats.total_time_saved`, `self.stats.encoding_cache_misses`; calls `self._make_key`, `self._encoding_cache.pop`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 330, "end_line": 358, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L330-L358", "decorators": [], "calls": [ "self._make_key", "self._encoding_cache.pop", "logger.debug" ], "state_reads": [ "self.enabled", "self._make_key", "self._encoding_cache", "self._encoding_cache.pop", "self.stats" ], "state_writes": [ "self.stats.encoding_cache_hits", "self.stats.total_time_saved", "self.stats.encoding_cache_misses" ], "raises": [], "return_expressions": [ "None", "entry" ], "awaits": false, "yields": false }, { "name": "set_encoding_cache", "qualname": "VisionEmbeddingCache.set_encoding_cache", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.set_encoding_cache", "kind": "method", "signature": "def set_encoding_cache(self, images: List[str], prompt: str, logits: mx.array, first_token: int, logprobs: mx.array, encoding_time: float=0.0) -> None", "parameters": [ { "name": "images", "kind": "positional or keyword", "annotation": "List[str]", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "prompt", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logits", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "first_token", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "logprobs", "kind": "positional or keyword", "annotation": "mx.array", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "encoding_time", "kind": "positional or keyword", "annotation": "float", "default": "0.0", "required": false, "description": "Optional positional or keyword input; defaults to `0.0`." } ], "return_annotation": "None", "docstring": "Store vision encoding output in cache.", "summary": "Store vision encoding output in cache.", "implementation": "Method `VisionEmbeddingCache.set_encoding_cache` calls `self._make_key`, `len`, `next`, `iter`; returns `None`.", "documented": true, "public": true, "addressable": true, "line": 360, "end_line": 388, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L360-L388", "decorators": [], "calls": [ "self._make_key", "len", "next", "iter", "logger.debug", "EncodingCacheEntry" ], "state_reads": [ "self.enabled", "self._make_key", "self._encoding_cache", "self.max_encoding_entries" ], "state_writes": [], "raises": [], "return_expressions": [ "None" ], "awaits": false, "yields": false }, { "name": "get_stats", "qualname": "VisionEmbeddingCache.get_stats", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.get_stats", "kind": "method", "signature": "def get_stats(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get cache statistics.", "summary": "Get cache statistics.", "implementation": "Method `VisionEmbeddingCache.get_stats` calls `self.stats.to_dict`, `len`; returns `stats`.", "documented": true, "public": true, "addressable": true, "line": 392, "end_line": 398, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L392-L398", "decorators": [], "calls": [ "self.stats.to_dict", "len" ], "state_reads": [ "self.stats.to_dict", "self.stats", "self._pixel_cache", "self._pixel_only_cache", "self._encoding_cache" ], "state_writes": [], "raises": [], "return_expressions": [ "stats" ], "awaits": false, "yields": false }, { "name": "clear", "qualname": "VisionEmbeddingCache.clear", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.clear", "kind": "method", "signature": "def clear(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clear all caches and reset stats.", "summary": "Clear all caches and reset stats.", "implementation": "Method `VisionEmbeddingCache.clear` updates `self.stats`; calls `self._pixel_cache.clear`, `self._pixel_only_cache.clear`, `self._encoding_cache.clear`, `VisionCacheStats`.", "documented": true, "public": true, "addressable": true, "line": 400, "end_line": 405, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vision_embedding_cache.py#L400-L405", "decorators": [], "calls": [ "self._pixel_cache.clear", "self._pixel_only_cache.clear", "self._encoding_cache.clear", "VisionCacheStats" ], "state_reads": [ "self._pixel_cache.clear", "self._pixel_cache", "self._pixel_only_cache.clear", "self._pixel_only_cache", "self._encoding_cache.clear", "self._encoding_cache" ], "state_writes": [ "self.stats" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "VisionEmbeddingCache.__repr__", "full_name": "vllm_mlx.vision_embedding_cache.VisionEmbeddingCache.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `VisionEmbeddingCache.__repr__` calls `len`; returns `f' str", "parameters": [], "return_annotation": "str", "docstring": "Get the name of the Apple Silicon chip.", "summary": "Get the name of the Apple Silicon chip.", "implementation": "Function `_get_apple_chip_name` calls `subprocess.run`, `result.stdout.strip`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 24, "end_line": 35, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L24-L35", "decorators": [], "calls": [ "subprocess.run", "result.stdout.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "result.stdout.strip()", "'Apple Silicon'" ], "awaits": false, "yields": false }, { "name": "_get_unified_memory_size", "qualname": "_get_unified_memory_size", "full_name": "vllm_mlx.vllm_platform._get_unified_memory_size", "kind": "function", "signature": "def _get_unified_memory_size() -> int", "parameters": [], "return_annotation": "int", "docstring": "Get the total unified memory size in bytes.", "summary": "Get the total unified memory size in bytes.", "implementation": "Function `_get_unified_memory_size` calls `subprocess.run`, `int`, `result.stdout.strip`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 38, "end_line": 50, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L38-L50", "decorators": [], "calls": [ "subprocess.run", "int", "result.stdout.strip" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "int(result.stdout.strip())", "8 * 1024 * 1024 * 1024" ], "awaits": false, "yields": false }, { "name": "_is_mlx_available", "qualname": "_is_mlx_available", "full_name": "vllm_mlx.vllm_platform._is_mlx_available", "kind": "function", "signature": "def _is_mlx_available() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if MLX is available and working.", "summary": "Check if MLX is available and working.", "implementation": "Function `_is_mlx_available` calls `mx.array`, `logger.debug`; has 2 explicit return paths.", "documented": true, "public": false, "addressable": true, "line": 53, "end_line": 63, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L53-L63", "decorators": [], "calls": [ "mx.array", "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True", "False" ], "awaits": false, "yields": false }, { "name": "_is_apple_silicon", "qualname": "_is_apple_silicon", "full_name": "vllm_mlx.vllm_platform._is_apple_silicon", "kind": "function", "signature": "def _is_apple_silicon() -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Check if running on Apple Silicon.", "summary": "Check if running on Apple Silicon.", "implementation": "Function `_is_apple_silicon` calls `platform.machine`; returns `sys.platform == 'darwin' and platform.machine() == 'arm64'`.", "documented": true, "public": false, "addressable": true, "line": 66, "end_line": 68, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L66-L68", "decorators": [], "calls": [ "platform.machine" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "sys.platform == 'darwin' and platform.machine() == 'arm64'" ], "awaits": false, "yields": false }, { "name": "MLXPlatform", "qualname": "MLXPlatform", "full_name": "vllm_mlx.vllm_platform.MLXPlatform", "kind": "class", "signature": "class MLXPlatform", "parameters": [], "return_annotation": "MLXPlatform", "docstring": "Platform implementation for Apple Silicon using MLX.\n\nThis platform uses Apple's MLX framework for GPU-accelerated\ninference on Apple Silicon Macs. It integrates with mlx-lm for\nLLM inference and mlx-vlm for vision-language models.\n\nKey features:\n- Unified memory model (no CPU<->GPU transfers)\n- Native Metal GPU acceleration\n- Optimized kernels for Apple Silicon\n- Support for quantized models (4-bit, 8-bit)", "summary": "Platform implementation for Apple Silicon using MLX.", "implementation": "Class `MLXPlatform` declares 30 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 71, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L71-L351", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "_enum", "qualname": "MLXPlatform._enum", "full_name": "vllm_mlx.vllm_platform.MLXPlatform._enum", "kind": "method", "signature": "def _enum(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "", "summary": "Method `MLXPlatform._enum` returns `PlatformEnum.OOT`.", "implementation": "Method `MLXPlatform._enum` returns `PlatformEnum.OOT`.", "documented": false, "public": false, "addressable": true, "line": 90, "end_line": 93, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L90-L93", "decorators": [ "property" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "PlatformEnum.OOT" ], "awaits": false, "yields": false }, { "name": "supported_dtypes", "qualname": "MLXPlatform.supported_dtypes", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.supported_dtypes", "kind": "method", "signature": "def supported_dtypes(self) -> list[torch.dtype]", "parameters": [], "return_annotation": "list[torch.dtype]", "docstring": "Return supported dtypes for MLX.", "summary": "Return supported dtypes for MLX.", "implementation": "Method `MLXPlatform.supported_dtypes` calls `mx.array`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 122, "end_line": 133, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L122-L133", "decorators": [ "property" ], "calls": [ "mx.array" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "[torch.bfloat16, torch.float16, torch.float32]", "[torch.float16, torch.float32]" ], "awaits": false, "yields": false }, { "name": "is_cuda", "qualname": "MLXPlatform.is_cuda", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_cuda", "kind": "method", "signature": "def is_cuda(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``False`` because this platform does not use CUDA.", "summary": "Return ``False`` because this platform does not use CUDA.", "implementation": "Method `MLXPlatform.is_cuda` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 135, "end_line": 138, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L135-L138", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "is_rocm", "qualname": "MLXPlatform.is_rocm", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_rocm", "kind": "method", "signature": "def is_rocm(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``False`` because this platform does not use ROCm.", "summary": "Return ``False`` because this platform does not use ROCm.", "implementation": "Method `MLXPlatform.is_rocm` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 140, "end_line": 143, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L140-L143", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "is_tpu", "qualname": "MLXPlatform.is_tpu", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_tpu", "kind": "method", "signature": "def is_tpu(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``False`` because this platform is not a TPU backend.", "summary": "Return ``False`` because this platform is not a TPU backend.", "implementation": "Method `MLXPlatform.is_tpu` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 145, "end_line": 148, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L145-L148", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "is_xpu", "qualname": "MLXPlatform.is_xpu", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_xpu", "kind": "method", "signature": "def is_xpu(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``False`` because this platform does not use Intel XPU.", "summary": "Return ``False`` because this platform does not use Intel XPU.", "implementation": "Method `MLXPlatform.is_xpu` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 150, "end_line": 153, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L150-L153", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "is_cpu", "qualname": "MLXPlatform.is_cpu", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_cpu", "kind": "method", "signature": "def is_cpu(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``False`` because MLX targets Apple GPU acceleration here.", "summary": "Return ``False`` because MLX targets Apple GPU acceleration here.", "implementation": "Method `MLXPlatform.is_cpu` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 155, "end_line": 158, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L155-L158", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "is_mlx", "qualname": "MLXPlatform.is_mlx", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_mlx", "kind": "method", "signature": "def is_mlx(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``True`` to identify the MLX platform plugin.", "summary": "Return ``True`` to identify the MLX platform plugin.", "implementation": "Method `MLXPlatform.is_mlx` returns `True`.", "documented": true, "public": true, "addressable": true, "line": 160, "end_line": 163, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L160-L163", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True" ], "awaits": false, "yields": false }, { "name": "is_out_of_tree", "qualname": "MLXPlatform.is_out_of_tree", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_out_of_tree", "kind": "method", "signature": "def is_out_of_tree(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``True`` because MLX is registered as a vLLM plugin.", "summary": "Return ``True`` because MLX is registered as a vLLM plugin.", "implementation": "Method `MLXPlatform.is_out_of_tree` returns `True`.", "documented": true, "public": true, "addressable": true, "line": 165, "end_line": 168, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L165-L168", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "True" ], "awaits": false, "yields": false }, { "name": "is_cuda_alike", "qualname": "MLXPlatform.is_cuda_alike", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_cuda_alike", "kind": "method", "signature": "def is_cuda_alike(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``False`` because MLX does not implement CUDA semantics.", "summary": "Return ``False`` because MLX does not implement CUDA semantics.", "implementation": "Method `MLXPlatform.is_cuda_alike` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 170, "end_line": 173, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L170-L173", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "is_sleep_mode_available", "qualname": "MLXPlatform.is_sleep_mode_available", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_sleep_mode_available", "kind": "method", "signature": "def is_sleep_mode_available(self) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Return ``False`` because vLLM sleep mode is unavailable on MLX.", "summary": "Return ``False`` because vLLM sleep mode is unavailable on MLX.", "implementation": "Method `MLXPlatform.is_sleep_mode_available` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 175, "end_line": 178, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L175-L178", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "get_device_name", "qualname": "MLXPlatform.get_device_name", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.get_device_name", "kind": "method", "signature": "def get_device_name(cls, device_id: int=0) -> str", "parameters": [ { "name": "device_id", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "str", "docstring": "Get the Apple Silicon chip name.", "summary": "Get the Apple Silicon chip name.", "implementation": "Method `MLXPlatform.get_device_name` calls `_get_apple_chip_name`; returns `_get_apple_chip_name()`.", "documented": true, "public": true, "addressable": true, "line": 181, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L181-L183", "decorators": [ "classmethod" ], "calls": [ "_get_apple_chip_name" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_get_apple_chip_name()" ], "awaits": false, "yields": false }, { "name": "get_device_uuid", "qualname": "MLXPlatform.get_device_uuid", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.get_device_uuid", "kind": "method", "signature": "def get_device_uuid(cls, device_id: int=0) -> str", "parameters": [ { "name": "device_id", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "str", "docstring": "Get device UUID (not applicable for MLX).", "summary": "Get device UUID (not applicable for MLX).", "implementation": "Method `MLXPlatform.get_device_uuid` returns `'mlx-0'`.", "documented": true, "public": true, "addressable": true, "line": 186, "end_line": 188, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L186-L188", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'mlx-0'" ], "awaits": false, "yields": false }, { "name": "get_device_total_memory", "qualname": "MLXPlatform.get_device_total_memory", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.get_device_total_memory", "kind": "method", "signature": "def get_device_total_memory(cls, device_id: int=0) -> int", "parameters": [ { "name": "device_id", "kind": "positional or keyword", "annotation": "int", "default": "0", "required": false, "description": "Optional positional or keyword input; defaults to `0`." } ], "return_annotation": "int", "docstring": "Get total unified memory in bytes.", "summary": "Get total unified memory in bytes.", "implementation": "Method `MLXPlatform.get_device_total_memory` calls `_get_unified_memory_size`; returns `_get_unified_memory_size()`.", "documented": true, "public": true, "addressable": true, "line": 191, "end_line": 193, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L191-L193", "decorators": [ "classmethod" ], "calls": [ "_get_unified_memory_size" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "_get_unified_memory_size()" ], "awaits": false, "yields": false }, { "name": "inference_mode", "qualname": "MLXPlatform.inference_mode", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.inference_mode", "kind": "method", "signature": "def inference_mode(cls)", "parameters": [], "return_annotation": "not annotated", "docstring": "Return inference mode context manager.", "summary": "Return inference mode context manager.", "implementation": "Method `MLXPlatform.inference_mode` calls `torch.no_grad`; returns `torch.no_grad()`.", "documented": true, "public": true, "addressable": true, "line": 196, "end_line": 200, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L196-L200", "decorators": [ "classmethod" ], "calls": [ "torch.no_grad" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "torch.no_grad()" ], "awaits": false, "yields": false }, { "name": "set_device", "qualname": "MLXPlatform.set_device", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.set_device", "kind": "method", "signature": "def set_device(cls, device: torch.device) -> None", "parameters": [ { "name": "device", "kind": "positional or keyword", "annotation": "torch.device", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Set the device (no-op for MLX, uses default device).", "summary": "Set the device (no-op for MLX, uses default device).", "implementation": "Method `MLXPlatform.set_device` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 203, "end_line": 206, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L203-L206", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "seed_everything", "qualname": "MLXPlatform.seed_everything", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.seed_everything", "kind": "method", "signature": "def seed_everything(cls, seed: int | None=None) -> None", "parameters": [ { "name": "seed", "kind": "positional or keyword", "annotation": "int | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Set random seeds for reproducibility.", "summary": "Set random seeds for reproducibility.", "implementation": "Method `MLXPlatform.seed_everything` calls `random.seed`, `np.random.seed`, `torch.manual_seed`, `mx.random.seed`.", "documented": true, "public": true, "addressable": true, "line": 209, "end_line": 225, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L209-L225", "decorators": [ "classmethod" ], "calls": [ "random.seed", "np.random.seed", "torch.manual_seed", "mx.random.seed" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "import_kernels", "qualname": "MLXPlatform.import_kernels", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.import_kernels", "kind": "method", "signature": "def import_kernels(cls) -> None", "parameters": [], "return_annotation": "None", "docstring": "Import MLX kernels (no custom C kernels).", "summary": "Import MLX kernels (no custom C kernels).", "implementation": "Method `MLXPlatform.import_kernels` contains no state mutation, call, raise, return, await, or yield.", "documented": true, "public": true, "addressable": true, "line": 228, "end_line": 231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L228-L231", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_attn_backend_cls", "qualname": "MLXPlatform.get_attn_backend_cls", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.get_attn_backend_cls", "kind": "method", "signature": "def get_attn_backend_cls(cls, selected_backend, head_size: int, dtype: torch.dtype, kv_cache_dtype, block_size: int, use_mla: bool, has_sink: bool, use_sparse: bool, attn_type: str | None=None) -> str", "parameters": [ { "name": "selected_backend", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "head_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "dtype", "kind": "positional or keyword", "annotation": "torch.dtype", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "kv_cache_dtype", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "block_size", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "use_mla", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "has_sink", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "use_sparse", "kind": "positional or keyword", "annotation": "bool", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "attn_type", "kind": "positional or keyword", "annotation": "str | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "str", "docstring": "Return MLX attention backend class path.", "summary": "Return MLX attention backend class path.", "implementation": "Method `MLXPlatform.get_attn_backend_cls` returns `'vllm_mlx.attention.MLXAttentionBackend'`.", "documented": true, "public": true, "addressable": true, "line": 234, "end_line": 248, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L234-L248", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'vllm_mlx.attention.MLXAttentionBackend'" ], "awaits": false, "yields": false }, { "name": "check_and_update_config", "qualname": "MLXPlatform.check_and_update_config", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.check_and_update_config", "kind": "method", "signature": "def check_and_update_config(cls, vllm_config: 'VllmConfig') -> None", "parameters": [ { "name": "vllm_config", "kind": "positional or keyword", "annotation": "'VllmConfig'", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Check and update vLLM configuration for MLX.", "summary": "Check and update vLLM configuration for MLX.", "implementation": "Method `MLXPlatform.check_and_update_config` calls `logger.info`, `_get_apple_chip_name`, `_get_unified_memory_size`, `hasattr`.", "documented": true, "public": true, "addressable": true, "line": 251, "end_line": 280, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L251-L280", "decorators": [ "classmethod" ], "calls": [ "logger.info", "_get_apple_chip_name", "_get_unified_memory_size", "hasattr", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "verify_model_arch", "qualname": "MLXPlatform.verify_model_arch", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.verify_model_arch", "kind": "method", "signature": "def verify_model_arch(cls, model_arch: str) -> None", "parameters": [ { "name": "model_arch", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Verify model architecture is supported on MLX.", "summary": "Verify model architecture is supported on MLX.", "implementation": "Method `MLXPlatform.verify_model_arch` calls `hint.lower`, `model_arch.lower`, `logger.warning`.", "documented": true, "public": true, "addressable": true, "line": 283, "end_line": 294, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L283-L294", "decorators": [ "classmethod" ], "calls": [ "hint.lower", "model_arch.lower", "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "verify_quantization", "qualname": "MLXPlatform.verify_quantization", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.verify_quantization", "kind": "method", "signature": "def verify_quantization(cls, quant: str) -> None", "parameters": [ { "name": "quant", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Verify quantization method is supported.", "summary": "Verify quantization method is supported.", "implementation": "Method `MLXPlatform.verify_quantization` calls `ValueError`; can raise `ValueError`.", "documented": true, "public": true, "addressable": true, "line": 297, "end_line": 304, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L297-L304", "decorators": [ "classmethod" ], "calls": [ "ValueError" ], "state_reads": [], "state_writes": [], "raises": [ "ValueError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "is_pin_memory_available", "qualname": "MLXPlatform.is_pin_memory_available", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.is_pin_memory_available", "kind": "method", "signature": "def is_pin_memory_available(cls) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Pin memory not needed with unified memory.", "summary": "Pin memory not needed with unified memory.", "implementation": "Method `MLXPlatform.is_pin_memory_available` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 307, "end_line": 309, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L307-L309", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "get_current_memory_usage", "qualname": "MLXPlatform.get_current_memory_usage", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.get_current_memory_usage", "kind": "method", "signature": "def get_current_memory_usage(cls, device=None) -> float", "parameters": [ { "name": "device", "kind": "positional or keyword", "annotation": "not annotated", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "float", "docstring": "Get current memory usage in bytes.", "summary": "Get current memory usage in bytes.", "implementation": "Method `MLXPlatform.get_current_memory_usage` calls `psutil.Process`, `float`, `process.memory_info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 312, "end_line": 323, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L312-L323", "decorators": [ "classmethod" ], "calls": [ "psutil.Process", "float", "process.memory_info" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "float(process.memory_info().rss)", "0.0" ], "awaits": false, "yields": false }, { "name": "supports_fp8", "qualname": "MLXPlatform.supports_fp8", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.supports_fp8", "kind": "method", "signature": "def supports_fp8(cls) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "FP8 not supported on MLX.", "summary": "FP8 not supported on MLX.", "implementation": "Method `MLXPlatform.supports_fp8` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 326, "end_line": 328, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L326-L328", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "use_custom_allreduce", "qualname": "MLXPlatform.use_custom_allreduce", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.use_custom_allreduce", "kind": "method", "signature": "def use_custom_allreduce(cls) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Custom allreduce not available.", "summary": "Custom allreduce not available.", "implementation": "Method `MLXPlatform.use_custom_allreduce` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 331, "end_line": 333, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L331-L333", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "support_static_graph_mode", "qualname": "MLXPlatform.support_static_graph_mode", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.support_static_graph_mode", "kind": "method", "signature": "def support_static_graph_mode(cls) -> bool", "parameters": [], "return_annotation": "bool", "docstring": "Static graph mode (CUDA graphs) not supported.", "summary": "Static graph mode (CUDA graphs) not supported.", "implementation": "Method `MLXPlatform.support_static_graph_mode` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 336, "end_line": 338, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L336-L338", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "get_device_communicator_cls", "qualname": "MLXPlatform.get_device_communicator_cls", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.get_device_communicator_cls", "kind": "method", "signature": "def get_device_communicator_cls(cls) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return the communicator class for distributed.", "summary": "Return the communicator class for distributed.", "implementation": "Method `MLXPlatform.get_device_communicator_cls` returns `'vllm_mlx.distributed.MLXCommunicator'`.", "documented": true, "public": true, "addressable": true, "line": 341, "end_line": 343, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L341-L343", "decorators": [ "classmethod" ], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "'vllm_mlx.distributed.MLXCommunicator'" ], "awaits": false, "yields": false }, { "name": "get_punica_wrapper", "qualname": "MLXPlatform.get_punica_wrapper", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.get_punica_wrapper", "kind": "method", "signature": "def get_punica_wrapper(cls) -> str", "parameters": [], "return_annotation": "str", "docstring": "Return LoRA wrapper (not yet implemented for MLX).", "summary": "Return LoRA wrapper (not yet implemented for MLX).", "implementation": "Method `MLXPlatform.get_punica_wrapper` calls `NotImplementedError`; can raise `NotImplementedError`.", "documented": true, "public": true, "addressable": true, "line": 346, "end_line": 348, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L346-L348", "decorators": [ "classmethod" ], "calls": [ "NotImplementedError" ], "state_reads": [], "state_writes": [], "raises": [ "NotImplementedError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "MLXPlatform.__repr__", "full_name": "vllm_mlx.vllm_platform.MLXPlatform.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `MLXPlatform.__repr__` returns `f''`.", "implementation": "Method `MLXPlatform.__repr__` returns `f''`.", "documented": false, "public": false, "addressable": true, "line": 350, "end_line": 351, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/vllm_platform.py#L350-L351", "decorators": [], "calls": [], "state_reads": [ "self.device_name" ], "state_writes": [], "raises": [], "return_expressions": [ "f''" ], "awaits": false, "yields": false } ] }, { "name": "vllm_mlx.worker", "path": "vllm_mlx/worker.py", "page_path": "reference/api/vllm_mlx/worker.md", "docstring": "MLX Worker for vLLM.\n\nThis module implements a vLLM worker that uses Apple's MLX framework\nfor model execution on Apple Silicon.", "summary": "MLX Worker for vLLM.", "line_count": 278, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L1-L278", "members": [ "logger", "MLXWorker" ], "symbols": [ { "name": "MLXWorker", "qualname": "MLXWorker", "full_name": "vllm_mlx.worker.MLXWorker", "kind": "class", "signature": "class MLXWorker", "parameters": [ { "name": "vllm_config", "kind": "positional or keyword", "annotation": "'VllmConfig'", "default": "", "required": true, "description": "Complete vLLM configuration" }, { "name": "local_rank", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Local device index (usually 0 for single GPU)" }, { "name": "rank", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Global rank in distributed setup" }, { "name": "distributed_init_method", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Distributed initialization method" }, { "name": "is_driver_worker", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether this worker handles driver responsibilities" } ], "return_annotation": "MLXWorker", "docstring": "Worker implementation for MLX-based inference on Apple Silicon.\n\nThis worker uses mlx-lm for model loading and inference, providing\nnative Apple Silicon GPU acceleration through Metal.\n\nUnlike CUDA workers that use PyTorch with CUDA, this worker:\n- Uses MLX arrays instead of PyTorch tensors for model weights\n- Leverages unified memory (no CPU<->GPU transfers needed)\n- Uses Metal-optimized kernels for attention and other operations", "summary": "Worker implementation for MLX-based inference on Apple Silicon.", "implementation": "Class `MLXWorker` declares 21 direct member(s).", "documented": true, "public": true, "addressable": true, "line": 23, "end_line": 278, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L23-L278", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__init__", "qualname": "MLXWorker.__init__", "full_name": "vllm_mlx.worker.MLXWorker.__init__", "kind": "method", "signature": "def __init__(self, vllm_config: 'VllmConfig', local_rank: int, rank: int, distributed_init_method: str, is_driver_worker: bool=False) -> None", "parameters": [ { "name": "vllm_config", "kind": "positional or keyword", "annotation": "'VllmConfig'", "default": "", "required": true, "description": "Complete vLLM configuration" }, { "name": "local_rank", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Local device index (usually 0 for single GPU)" }, { "name": "rank", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Global rank in distributed setup" }, { "name": "distributed_init_method", "kind": "positional or keyword", "annotation": "str", "default": "", "required": true, "description": "Distributed initialization method" }, { "name": "is_driver_worker", "kind": "positional or keyword", "annotation": "bool", "default": "False", "required": false, "description": "Whether this worker handles driver responsibilities" } ], "return_annotation": "None", "docstring": "Initialize MLX worker.\n\nArgs:\n vllm_config: Complete vLLM configuration\n local_rank: Local device index (usually 0 for single GPU)\n rank: Global rank in distributed setup\n distributed_init_method: Distributed initialization method\n is_driver_worker: Whether this worker handles driver responsibilities", "summary": "Initialize MLX worker.", "implementation": "Method `MLXWorker.__init__` updates `self.vllm_config`, `self.model_config`, `self.cache_config`, `self.parallel_config`; calls `torch.device`, `logger.info`.", "documented": true, "public": false, "addressable": true, "line": 36, "end_line": 75, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L36-L75", "decorators": [], "calls": [ "torch.device", "logger.info" ], "state_reads": [], "state_writes": [ "self.vllm_config", "self.model_config", "self.cache_config", "self.parallel_config", "self.scheduler_config", "self.device_config", "self.load_config", "self.local_rank", "self.rank", "self.distributed_init_method", "self.is_driver_worker", "self.model", "self.tokenizer", "self.model_runner", "self.device" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "init_device", "qualname": "MLXWorker.init_device", "full_name": "vllm_mlx.worker.MLXWorker.init_device", "kind": "method", "signature": "def init_device(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Initialize MLX device and verify it's working.", "summary": "Initialize MLX device and verify it's working.", "implementation": "Method `MLXWorker.init_device` updates `self.model_runner`; calls `mx.default_device`, `logger.info`, `get_mlx_device_info`, `MLXModelRunner`; can raise `ImportError`.", "documented": true, "public": true, "addressable": true, "line": 77, "end_line": 103, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L77-L103", "decorators": [], "calls": [ "mx.default_device", "logger.info", "get_mlx_device_info", "MLXModelRunner", "ImportError" ], "state_reads": [ "self.vllm_config" ], "state_writes": [ "self.model_runner" ], "raises": [ "ImportError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "load_model", "qualname": "MLXWorker.load_model", "full_name": "vllm_mlx.worker.MLXWorker.load_model", "kind": "method", "signature": "def load_model(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Load model using mlx-lm.", "summary": "Load model using mlx-lm.", "implementation": "Method `MLXWorker.load_model` calls `RuntimeError`, `self.model_runner.load_model`, `logger.info`; can raise `RuntimeError`.", "documented": true, "public": true, "addressable": true, "line": 105, "end_line": 111, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L105-L111", "decorators": [], "calls": [ "RuntimeError", "self.model_runner.load_model", "logger.info" ], "state_reads": [ "self.model_runner", "self.model_runner.load_model", "self.model_config.model", "self.model_config" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "determine_available_memory", "qualname": "MLXWorker.determine_available_memory", "full_name": "vllm_mlx.worker.MLXWorker.determine_available_memory", "kind": "method", "signature": "def determine_available_memory(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Determine available memory for KV cache.\n\nOn Apple Silicon with unified memory, we use a portion of system RAM.", "summary": "Determine available memory for KV cache.", "implementation": "Method `MLXWorker.determine_available_memory` calls `subprocess.run`, `int`, `result.stdout.strip`, `logger.info`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 113, "end_line": 143, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L113-L143", "decorators": [], "calls": [ "subprocess.run", "int", "result.stdout.strip", "logger.info", "logger.warning" ], "state_reads": [ "self.cache_config.gpu_memory_utilization", "self.cache_config" ], "state_writes": [], "raises": [], "return_expressions": [ "available", "4 * 1024 * 1024 * 1024" ], "awaits": false, "yields": false }, { "name": "initialize_cache", "qualname": "MLXWorker.initialize_cache", "full_name": "vllm_mlx.worker.MLXWorker.initialize_cache", "kind": "method", "signature": "def initialize_cache(self, num_gpu_blocks: int, num_cpu_blocks: int) -> None", "parameters": [ { "name": "num_gpu_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." }, { "name": "num_cpu_blocks", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "None", "docstring": "Initialize KV cache with the given size.", "summary": "Initialize KV cache with the given size.", "implementation": "Method `MLXWorker.initialize_cache` updates `self.cache_config.num_gpu_blocks`, `self.cache_config.num_cpu_blocks`; calls `self.model_runner.initialize_cache`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 145, "end_line": 153, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L145-L153", "decorators": [], "calls": [ "self.model_runner.initialize_cache", "logger.info" ], "state_reads": [ "self.cache_config", "self.model_runner", "self.model_runner.initialize_cache" ], "state_writes": [ "self.cache_config.num_gpu_blocks", "self.cache_config.num_cpu_blocks" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "get_kv_cache_spec", "qualname": "MLXWorker.get_kv_cache_spec", "full_name": "vllm_mlx.worker.MLXWorker.get_kv_cache_spec", "kind": "method", "signature": "def get_kv_cache_spec(self) -> dict", "parameters": [], "return_annotation": "dict", "docstring": "Get KV cache specification.", "summary": "Get KV cache specification.", "implementation": "Method `MLXWorker.get_kv_cache_spec` calls `self.model_runner.get_kv_cache_spec`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 155, "end_line": 159, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L155-L159", "decorators": [], "calls": [ "self.model_runner.get_kv_cache_spec" ], "state_reads": [ "self.model_runner", "self.model_runner.get_kv_cache_spec" ], "state_writes": [], "raises": [], "return_expressions": [ "self.model_runner.get_kv_cache_spec()", "{}" ], "awaits": false, "yields": false }, { "name": "compile_or_warm_up_model", "qualname": "MLXWorker.compile_or_warm_up_model", "full_name": "vllm_mlx.worker.MLXWorker.compile_or_warm_up_model", "kind": "method", "signature": "def compile_or_warm_up_model(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Warm up model for inference.", "summary": "Warm up model for inference.", "implementation": "Method `MLXWorker.compile_or_warm_up_model` calls `self.model_runner.warm_up`, `logger.info`.", "documented": true, "public": true, "addressable": true, "line": 161, "end_line": 165, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L161-L165", "decorators": [], "calls": [ "self.model_runner.warm_up", "logger.info" ], "state_reads": [ "self.model_runner", "self.model_runner.warm_up" ], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "execute_model", "qualname": "MLXWorker.execute_model", "full_name": "vllm_mlx.worker.MLXWorker.execute_model", "kind": "method", "signature": "def execute_model(self, scheduler_output: 'SchedulerOutput') -> 'ModelRunnerOutput | None'", "parameters": [ { "name": "scheduler_output", "kind": "positional or keyword", "annotation": "'SchedulerOutput'", "default": "", "required": true, "description": "Contains requests to process" } ], "return_annotation": "'ModelRunnerOutput | None'", "docstring": "Execute model inference for the given scheduler output.\n\nArgs:\n scheduler_output: Contains requests to process\n\nReturns:\n ModelRunnerOutput with generation results", "summary": "Execute model inference for the given scheduler output.", "implementation": "Method `MLXWorker.execute_model` calls `RuntimeError`, `self.model_runner.execute_model`; can raise `RuntimeError`; returns `self.model_runner.execute_model(scheduler_output)`.", "documented": true, "public": true, "addressable": true, "line": 167, "end_line": 183, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L167-L183", "decorators": [], "calls": [ "RuntimeError", "self.model_runner.execute_model" ], "state_reads": [ "self.model_runner", "self.model_runner.execute_model" ], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [ "self.model_runner.execute_model(scheduler_output)" ], "awaits": false, "yields": false }, { "name": "get_model", "qualname": "MLXWorker.get_model", "full_name": "vllm_mlx.worker.MLXWorker.get_model", "kind": "method", "signature": "def get_model(self)", "parameters": [], "return_annotation": "not annotated", "docstring": "Get the underlying model.", "summary": "Get the underlying model.", "implementation": "Method `MLXWorker.get_model` has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 185, "end_line": 189, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L185-L189", "decorators": [], "calls": [], "state_reads": [ "self.model_runner", "self.model_runner.model" ], "state_writes": [], "raises": [], "return_expressions": [ "self.model_runner.model", "None" ], "awaits": false, "yields": false }, { "name": "check_health", "qualname": "MLXWorker.check_health", "full_name": "vllm_mlx.worker.MLXWorker.check_health", "kind": "method", "signature": "def check_health(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Check worker health.", "summary": "Check worker health.", "implementation": "Method `MLXWorker.check_health` calls `mx.array`, `mx.sum(test).item`, `mx.sum`, `RuntimeError`; can raise `RuntimeError`.", "documented": true, "public": true, "addressable": true, "line": 191, "end_line": 200, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L191-L200", "decorators": [], "calls": [ "mx.array", "mx.sum(test).item", "mx.sum", "RuntimeError" ], "state_reads": [], "state_writes": [], "raises": [ "RuntimeError" ], "return_expressions": [], "awaits": false, "yields": false }, { "name": "shutdown", "qualname": "MLXWorker.shutdown", "full_name": "vllm_mlx.worker.MLXWorker.shutdown", "kind": "method", "signature": "def shutdown(self) -> None", "parameters": [], "return_annotation": "None", "docstring": "Clean up resources.", "summary": "Clean up resources.", "implementation": "Method `MLXWorker.shutdown` updates `self.model`, `self.tokenizer`, `self.model_runner`; calls `logger.info`, `mx.clear_cache`, `gc.collect`.", "documented": true, "public": true, "addressable": true, "line": 202, "end_line": 219, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L202-L219", "decorators": [], "calls": [ "logger.info", "mx.clear_cache", "gc.collect" ], "state_reads": [], "state_writes": [ "self.model", "self.tokenizer", "self.model_runner" ], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "add_lora", "qualname": "MLXWorker.add_lora", "full_name": "vllm_mlx.worker.MLXWorker.add_lora", "kind": "method", "signature": "def add_lora(self, lora_request) -> bool", "parameters": [ { "name": "lora_request", "kind": "positional or keyword", "annotation": "not annotated", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Report that dynamically adding a LoRA adapter is unsupported.", "summary": "Report that dynamically adding a LoRA adapter is unsupported.", "implementation": "Method `MLXWorker.add_lora` calls `logger.warning`; returns `False`.", "documented": true, "public": true, "addressable": true, "line": 222, "end_line": 226, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L222-L226", "decorators": [], "calls": [ "logger.warning" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "remove_lora", "qualname": "MLXWorker.remove_lora", "full_name": "vllm_mlx.worker.MLXWorker.remove_lora", "kind": "method", "signature": "def remove_lora(self, lora_id: int) -> bool", "parameters": [ { "name": "lora_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Report that dynamically removing a LoRA adapter is unsupported.", "summary": "Report that dynamically removing a LoRA adapter is unsupported.", "implementation": "Method `MLXWorker.remove_lora` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 228, "end_line": 231, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L228-L231", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "pin_lora", "qualname": "MLXWorker.pin_lora", "full_name": "vllm_mlx.worker.MLXWorker.pin_lora", "kind": "method", "signature": "def pin_lora(self, lora_id: int) -> bool", "parameters": [ { "name": "lora_id", "kind": "positional or keyword", "annotation": "int", "default": "", "required": true, "description": "Required positional or keyword input." } ], "return_annotation": "bool", "docstring": "Report that pinning a LoRA adapter is unsupported.", "summary": "Report that pinning a LoRA adapter is unsupported.", "implementation": "Method `MLXWorker.pin_lora` returns `False`.", "documented": true, "public": true, "addressable": true, "line": 233, "end_line": 236, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L233-L236", "decorators": [], "calls": [], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "False" ], "awaits": false, "yields": false }, { "name": "list_loras", "qualname": "MLXWorker.list_loras", "full_name": "vllm_mlx.worker.MLXWorker.list_loras", "kind": "method", "signature": "def list_loras(self) -> set[int]", "parameters": [], "return_annotation": "set[int]", "docstring": "Return the empty set because runtime LoRA adapters are unsupported.", "summary": "Return the empty set because runtime LoRA adapters are unsupported.", "implementation": "Method `MLXWorker.list_loras` calls `set`; returns `set()`.", "documented": true, "public": true, "addressable": true, "line": 238, "end_line": 241, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L238-L241", "decorators": [], "calls": [ "set" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [ "set()" ], "awaits": false, "yields": false }, { "name": "sleep", "qualname": "MLXWorker.sleep", "full_name": "vllm_mlx.worker.MLXWorker.sleep", "kind": "method", "signature": "def sleep(self, level: int=1) -> None", "parameters": [ { "name": "level", "kind": "positional or keyword", "annotation": "int", "default": "1", "required": false, "description": "Optional positional or keyword input; defaults to `1`." } ], "return_annotation": "None", "docstring": "Leave the worker active because MLX unified memory has no sleep mode.", "summary": "Leave the worker active because MLX unified memory has no sleep mode.", "implementation": "Method `MLXWorker.sleep` calls `logger.debug`.", "documented": true, "public": true, "addressable": true, "line": 244, "end_line": 247, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L244-L247", "decorators": [], "calls": [ "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "wake_up", "qualname": "MLXWorker.wake_up", "full_name": "vllm_mlx.worker.MLXWorker.wake_up", "kind": "method", "signature": "def wake_up(self, tags: list[str] | None=None) -> None", "parameters": [ { "name": "tags", "kind": "positional or keyword", "annotation": "list[str] | None", "default": "None", "required": false, "description": "Optional positional or keyword input; defaults to `None`." } ], "return_annotation": "None", "docstring": "Perform no work because the MLX worker never enters sleep mode.", "summary": "Perform no work because the MLX worker never enters sleep mode.", "implementation": "Method `MLXWorker.wake_up` calls `logger.debug`.", "documented": true, "public": true, "addressable": true, "line": 249, "end_line": 252, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L249-L252", "decorators": [], "calls": [ "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "vocab_size", "qualname": "MLXWorker.vocab_size", "full_name": "vllm_mlx.worker.MLXWorker.vocab_size", "kind": "method", "signature": "def vocab_size(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get vocabulary size.", "summary": "Get vocabulary size.", "implementation": "Method `MLXWorker.vocab_size` calls `self.model_config.get_vocab_size`; returns `self.model_config.get_vocab_size()`.", "documented": true, "public": true, "addressable": true, "line": 255, "end_line": 257, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L255-L257", "decorators": [ "property" ], "calls": [ "self.model_config.get_vocab_size" ], "state_reads": [ "self.model_config.get_vocab_size", "self.model_config" ], "state_writes": [], "raises": [], "return_expressions": [ "self.model_config.get_vocab_size()" ], "awaits": false, "yields": false }, { "name": "get_cache_block_size_bytes", "qualname": "MLXWorker.get_cache_block_size_bytes", "full_name": "vllm_mlx.worker.MLXWorker.get_cache_block_size_bytes", "kind": "method", "signature": "def get_cache_block_size_bytes(self) -> int", "parameters": [], "return_annotation": "int", "docstring": "Get size of a cache block in bytes.", "summary": "Get size of a cache block in bytes.", "implementation": "Method `MLXWorker.get_cache_block_size_bytes` calls `self.model_runner.get_cache_block_size_bytes`, `self.model_config.get_head_size`, `self.model_config.get_num_kv_heads`, `self.model_config.get_num_layers`; has 2 explicit return paths.", "documented": true, "public": true, "addressable": true, "line": 259, "end_line": 271, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L259-L271", "decorators": [], "calls": [ "self.model_runner.get_cache_block_size_bytes", "self.model_config.get_head_size", "self.model_config.get_num_kv_heads", "self.model_config.get_num_layers" ], "state_reads": [ "self.model_runner", "self.model_runner.get_cache_block_size_bytes", "self.model_config.get_head_size", "self.model_config", "self.model_config.get_num_kv_heads", "self.parallel_config", "self.model_config.get_num_layers", "self.cache_config.block_size", "self.cache_config" ], "state_writes": [], "raises": [], "return_expressions": [ "self.model_runner.get_cache_block_size_bytes()", "2 * block_size * num_layers * num_heads * head_size * 2" ], "awaits": false, "yields": false }, { "name": "profile", "qualname": "MLXWorker.profile", "full_name": "vllm_mlx.worker.MLXWorker.profile", "kind": "method", "signature": "def profile(self, is_start: bool=True) -> None", "parameters": [ { "name": "is_start", "kind": "positional or keyword", "annotation": "bool", "default": "True", "required": false, "description": "Optional positional or keyword input; defaults to `True`." } ], "return_annotation": "None", "docstring": "Profiling (not yet implemented for MLX).", "summary": "Profiling (not yet implemented for MLX).", "implementation": "Method `MLXWorker.profile` calls `logger.debug`.", "documented": true, "public": true, "addressable": true, "line": 273, "end_line": 275, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L273-L275", "decorators": [], "calls": [ "logger.debug" ], "state_reads": [], "state_writes": [], "raises": [], "return_expressions": [], "awaits": false, "yields": false }, { "name": "__repr__", "qualname": "MLXWorker.__repr__", "full_name": "vllm_mlx.worker.MLXWorker.__repr__", "kind": "method", "signature": "def __repr__(self) -> str", "parameters": [], "return_annotation": "str", "docstring": "", "summary": "Method `MLXWorker.__repr__` returns `f''`.", "implementation": "Method `MLXWorker.__repr__` returns `f''`.", "documented": false, "public": false, "addressable": true, "line": 277, "end_line": 278, "source_url": "https://github.com/waybarrios/vllm-mlx/blob/a69d47912bcb21d8fe04d48f75fa896b620ffcfa/vllm_mlx/worker.py#L277-L278", "decorators": [], "calls": [], "state_reads": [ "self.rank", "self.local_rank" ], "state_writes": [], "raises": [], "return_expressions": [ "f''" ], "awaits": false, "yields": false } ] } ] }