vllm.v1.worker.gpu.spec_decode.speculator_config

Per-speculator configuration.

Each speculator has a SpeculatorConfig subclass, in a config.py next to its V2 model runner implementation, which sets up SpeculativeConfig for it, describes its behavior to code outside the speculator (queried through SpeculativeConfig.speculator_config rather than by method name), and names its implementation class. Config modules must not import the speculators, as they're used in every process. Custom speculators are registered with register_speculative_method.

Classes:

• DraftModelSpeculatorConfig –

  A speculator whose draft model is loaded from the model checkpoint.

• SpeculatorConfig –

  Configuration of a kind of speculator.

Functions:

• get_custom_speculator_config_class –

  The config of a registered custom speculative method, or None.

• get_method_config_class –

  The config class for spec_config.method, regardless of the draft

• get_speculator_config_class –

  The config class for a set-up spec config.

• register_speculative_method –

  Register a custom speculative method, used with method=name.

• registered_speculative_methods –

  The names of the registered custom speculative methods.

DraftModelSpeculatorConfig

Bases: SpeculatorConfig

A speculator whose draft model is loaded from the model checkpoint.

Methods:

• resolve_num_speculative_tokens –

  Default or validate num_speculative_tokens from the checkpoint.

• transformers_backend_hf_overrides –

  HF config updates for a draft built by the Transformers modeling

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

class DraftModelSpeculatorConfig(SpeculatorConfig):
    """A speculator whose draft model is loaded from the `model` checkpoint."""

    # The draft model takes the target's positions, so its
    # max_position_embeddings is raised to the target's max_model_len where its
    # RoPE type allows.
    shares_target_positions: ClassVar[bool] = True

    @property
    def max_model_len(self) -> int | None:
        from vllm.config.speculative import (
            _rope_only_sizes_cache,
            _yarn_cache_positions,
        )

        draft = self.spec_config.draft_model_config
        if draft is None:
            return None
        if self.shares_target_positions:
            if _rope_only_sizes_cache(draft.hf_text_config):
                # Its max_position_embeddings was raised to the target's.
                return None
            if (positions := _yarn_cache_positions(draft.hf_text_config)) is not None:
                return positions
        return draft.max_model_len

    @classmethod
    def default_model(cls, spec: "SpeculativeConfig") -> str | None:
        raise ValueError(
            "num_speculative_tokens was provided but without speculative model."
        )

    @classmethod
    def configure(cls, spec: "SpeculativeConfig") -> None:
        from vllm.config import ModelConfig, SpeculativeConfig

        spec.prompt_lookup_max = 0
        spec.prompt_lookup_min = 0
        if spec.model is None:
            return

        target = spec.target_model_config
        # Callable target hf_overrides also apply to the draft; rope-relevant
        # dict keys (YaRN / RoPE) do too. Other dict patches stay target-only.
        draft_hf_overrides = SpeculativeConfig.compose_draft_hf_overrides(
            target.hf_overrides, cls.transformers_backend_hf_overrides(spec)
        )
        spec.draft_model_config = ModelConfig(
            model=spec.model,
            runner="draft",
            tokenizer=target.tokenizer,
            tokenizer_mode=target.tokenizer_mode,
            trust_remote_code=target.trust_remote_code,
            allowed_local_media_path=target.allowed_local_media_path,
            allowed_media_domains=target.allowed_media_domains,
            dtype=target.dtype,
            seed=target.seed,
            revision=spec.revision,
            code_revision=spec.code_revision,
            tokenizer_revision=target.tokenizer_revision,
            max_model_len=spec.max_model_len,  # type: ignore[arg-type]
            spec_target_max_model_len=target.max_model_len,
            quantization=spec.quantization,
            enforce_eager=target.enforce_eager,
            max_logprobs=target.max_logprobs,
            hf_overrides=draft_hf_overrides,
            config_format=target.config_format,
        )

        config_cls = _resolve_checkpoint_config_class(spec, cls)
        if spec.method is None:
            spec.method = config_cls.base_method  # type: ignore[assignment]

        if config_cls.shares_target_positions:
            SpeculativeConfig._maybe_override_draft_max_position_embeddings(
                spec.draft_model_config.hf_text_config, target.max_model_len
            )
        config_cls.adapt_draft_model_config(spec)
        config_cls.resolve_num_speculative_tokens(spec)
        if spec.num_speculative_tokens is None:
            raise ValueError(
                "A speculative model was provided, but "
                "`num_speculative_tokens` was not provided"
            )

        spec.draft_tensor_parallel_size = SpeculativeConfig._verify_and_get_draft_tp(
            spec.target_parallel_config, spec.draft_tensor_parallel_size
        )
        spec.draft_model_config.max_model_len = (
            SpeculativeConfig._maybe_override_draft_max_model_len(
                spec.max_model_len,
                spec.draft_model_config.max_model_len,
                target.max_model_len,
            )
        )
        # From the final draft config, so EP agrees with its expert count.
        spec.draft_parallel_config = SpeculativeConfig.create_draft_parallel_config(
            spec.target_parallel_config,
            spec.draft_tensor_parallel_size,
            draft_model_config=spec.draft_model_config,
        )

    @classmethod
    def transformers_backend_hf_overrides(
        cls, spec: "SpeculativeConfig"
    ) -> dict[str, Any] | None:
        """HF config updates for a draft built by the Transformers modeling
        backend, which replace the usual draft config rewrites, or None."""
        return None

    @classmethod
    def adapt_draft_model_config(cls, spec: "SpeculativeConfig") -> None:
        return

    @classmethod
    def resolve_num_speculative_tokens(cls, spec: "SpeculativeConfig") -> None:
        """Default or validate `num_speculative_tokens` from the checkpoint."""
        n_predict = getattr(spec.draft_model_config.hf_config, "n_predict", None)
        if n_predict is None:
            return
        if spec.num_speculative_tokens is None:
            # Default to max value defined in draft model config.
            spec.num_speculative_tokens = n_predict
        elif (
            spec.num_speculative_tokens > n_predict
            and spec.num_speculative_tokens % n_predict != 0
        ):
            # Ensure divisibility for MTP module reuse.
            raise ValueError(
                f"num_speculative_tokens:{spec.num_speculative_tokens}"
                f" must be divisible by {n_predict=}"
            )

resolve_num_speculative_tokens(spec) classmethod

Default or validate num_speculative_tokens from the checkpoint.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

@classmethod
def resolve_num_speculative_tokens(cls, spec: "SpeculativeConfig") -> None:
    """Default or validate `num_speculative_tokens` from the checkpoint."""
    n_predict = getattr(spec.draft_model_config.hf_config, "n_predict", None)
    if n_predict is None:
        return
    if spec.num_speculative_tokens is None:
        # Default to max value defined in draft model config.
        spec.num_speculative_tokens = n_predict
    elif (
        spec.num_speculative_tokens > n_predict
        and spec.num_speculative_tokens % n_predict != 0
    ):
        # Ensure divisibility for MTP module reuse.
        raise ValueError(
            f"num_speculative_tokens:{spec.num_speculative_tokens}"
            f" must be divisible by {n_predict=}"
        )

transformers_backend_hf_overrides(spec) classmethod

HF config updates for a draft built by the Transformers modeling backend, which replace the usual draft config rewrites, or None.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

@classmethod
def transformers_backend_hf_overrides(
    cls, spec: "SpeculativeConfig"
) -> dict[str, Any] | None:
    """HF config updates for a draft built by the Transformers modeling
    backend, which replace the usual draft config rewrites, or None."""
    return None

SpeculatorConfig

Configuration of a kind of speculator.

The defaults describe a speculator without a draft checkpoint or special requirements. A custom speculator can subclass this, or the config of the built-in speculator it behaves like (overriding speculator_class), whose setup and behavior it then inherits.

Methods:

• apply_options –

  Apply method-specific options to the draft model config.

• configure –

  Set up the draft model config (and related options).

• default_model –

  The model to use when it isn't given.

• hash_factors –

  Extra factors of SpeculativeConfig.compile_factors.

• matches –

  Whether the draft checkpoint is for this speculator, other than by

• max_num_draft_seqs –

  The most requests the draft forward pass fits in

• pcp_unsupported_reason –

  Why prefill context parallelism can't be used, or None.

• specialize –

  This config, or a more specific subclass for the settings.

Attributes:

• max_model_len (int | None) –

  The longest sequence the speculator can draft for, or None if

• max_num_new_slots_for_drafting (int) –

  See SpeculativeConfig.max_num_new_slots_for_drafting.

• num_lookahead_tokens (int) –

  See VllmConfig.num_lookahead_tokens.

• num_prefill_lookahead_tokens (int) –

  See VllmConfig.num_prefill_lookahead_tokens.

• uses_aux_hidden_states (bool) –

  Whether the target must also output intermediate hidden states.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

class SpeculatorConfig:
    """Configuration of a kind of speculator.

    The defaults describe a speculator without a draft checkpoint or special
    requirements. A custom speculator can subclass this, or the config of the
    built-in speculator it behaves like (overriding `speculator_class`), whose
    setup and behavior it then inherits.
    """

    # For in-tree speculators, the `SpeculativeConfig.method` that enables it.
    # For out-of-tree custom speculators, the method of the built-in one it subclasses.
    base_method: ClassVar[str | None] = None
    # Fully-qualified name of the V2 `BaseSpeculator` subclass, if any.
    speculator_class: ClassVar[str | None] = None
    # Method-specific `SpeculativeConfig` options that this speculator accepts
    # (e.g. "speculator_options" for a custom speculator's own options).
    supported_options: ClassVar[frozenset[str]] = frozenset()

    # Uses the target model's hidden states (EAGLE or EAGLE3-style).
    uses_target_hidden_states: ClassVar[bool] = False
    # Drafts in its own workspace lane.
    uses_separate_workspace_lane: ClassVar[bool] = False
    # For performance, prefers requests' full token history on the GPU instead of
    # UVA memory (e.g. ngram).
    prefer_dense_token_history: ClassVar[bool] = False
    # Proposes draft tokens (unlike e.g. hidden state extraction).
    proposes_draft_tokens: ClassVar[bool] = True
    adaptive_verification_uses_draft_logits: ClassVar[bool] = True
    # Generates all draft tokens in one forward pass (e.g. DFlash).
    parallel_drafting: ClassVar[bool] = False
    # The draft model has its own vocabulary, which must match the target's.
    draft_has_own_vocab: ClassVar[bool] = False

    supports_watermarking: ClassVar[bool] = False
    # Prefill context parallelism.
    supports_pcp: ClassVar[bool] = False

    # --- SpeculativeConfig setup ---

    @classmethod
    def matches(cls, spec: "SpeculativeConfig") -> bool:
        """Whether the draft checkpoint is for this speculator, other than by
        its model's declaration (e.g. by name). Used to detect the method."""
        return False

    @classmethod
    def specialize(cls, spec: "SpeculativeConfig") -> type["SpeculatorConfig"]:
        """This config, or a more specific subclass for the settings."""
        return cls

    @classmethod
    def default_model(cls, spec: "SpeculativeConfig") -> str | None:
        """The `model` to use when it isn't given."""
        return cls.base_method

    @classmethod
    def configure(cls, spec: "SpeculativeConfig") -> None:
        """Set up the draft model config (and related options)."""
        spec.prompt_lookup_max = 0
        spec.prompt_lookup_min = 0
        # No draft model; the target's config provides the vocab size etc.
        spec.draft_model_config = spec.target_model_config
        spec.draft_parallel_config = spec.target_parallel_config

    @classmethod
    def apply_options(cls, spec: "SpeculativeConfig") -> None:
        """Apply method-specific options to the draft model config."""
        return

    # --- Behavior ---

    def __init__(self, spec_config: "SpeculativeConfig"):
        # Read lazily, since the spec config may still be updated.
        self.spec_config = spec_config

    @property
    @final
    def num_speculative_tokens(self) -> int:
        return self.spec_config.num_speculative_tokens

    @property
    def uses_aux_hidden_states(self) -> bool:
        """Whether the target must also output intermediate hidden states."""
        return False

    @property
    def num_lookahead_tokens(self) -> int:
        """See `VllmConfig.num_lookahead_tokens`."""
        return 0

    @property
    def max_model_len(self) -> int | None:
        """The longest sequence the speculator can draft for, or None if
        unlimited. A smaller limit lowers overall max_model_len."""
        return None

    @property
    def num_prefill_lookahead_tokens(self) -> int:
        """See `VllmConfig.num_prefill_lookahead_tokens`."""
        # Hidden-state drafters read only the immediate next token.
        return 1 if self.uses_target_hidden_states else 0

    @property
    def max_num_new_slots_for_drafting(self) -> int:
        """See `SpeculativeConfig.max_num_new_slots_for_drafting`."""
        return 0

    def max_num_draft_seqs(self, max_num_batched_tokens: int) -> int | None:
        """The most requests the draft forward pass fits in
        max_num_batched_tokens, which limits max_num_active_seqs, or None if
        unlimited (e.g. when it's sized by the target's scheduled tokens)."""
        return None

    def hash_factors(self) -> list[Any]:
        """Extra factors of `SpeculativeConfig.compile_factors`."""
        return []

    def pcp_unsupported_reason(self, parallel_config: "ParallelConfig") -> str | None:
        """Why prefill context parallelism can't be used, or None."""
        if self.supports_pcp:
            return None
        return (
            "MRV2 PCP only supports DSpark or single-module MTP speculative decoding."
        )

max_model_len property

The longest sequence the speculator can draft for, or None if unlimited. A smaller limit lowers overall max_model_len.

max_num_new_slots_for_drafting property

See SpeculativeConfig.max_num_new_slots_for_drafting.

num_lookahead_tokens property

See VllmConfig.num_lookahead_tokens.

num_prefill_lookahead_tokens property

See VllmConfig.num_prefill_lookahead_tokens.

uses_aux_hidden_states property

Whether the target must also output intermediate hidden states.

apply_options(spec) classmethod

Apply method-specific options to the draft model config.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

@classmethod
def apply_options(cls, spec: "SpeculativeConfig") -> None:
    """Apply method-specific options to the draft model config."""
    return

configure(spec) classmethod

Set up the draft model config (and related options).

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

@classmethod
def configure(cls, spec: "SpeculativeConfig") -> None:
    """Set up the draft model config (and related options)."""
    spec.prompt_lookup_max = 0
    spec.prompt_lookup_min = 0
    # No draft model; the target's config provides the vocab size etc.
    spec.draft_model_config = spec.target_model_config
    spec.draft_parallel_config = spec.target_parallel_config

default_model(spec) classmethod

The model to use when it isn't given.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

@classmethod
def default_model(cls, spec: "SpeculativeConfig") -> str | None:
    """The `model` to use when it isn't given."""
    return cls.base_method

hash_factors()

Extra factors of SpeculativeConfig.compile_factors.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def hash_factors(self) -> list[Any]:
    """Extra factors of `SpeculativeConfig.compile_factors`."""
    return []

matches(spec) classmethod

Whether the draft checkpoint is for this speculator, other than by its model's declaration (e.g. by name). Used to detect the method.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

@classmethod
def matches(cls, spec: "SpeculativeConfig") -> bool:
    """Whether the draft checkpoint is for this speculator, other than by
    its model's declaration (e.g. by name). Used to detect the method."""
    return False

max_num_draft_seqs(max_num_batched_tokens)

The most requests the draft forward pass fits in max_num_batched_tokens, which limits max_num_active_seqs, or None if unlimited (e.g. when it's sized by the target's scheduled tokens).

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def max_num_draft_seqs(self, max_num_batched_tokens: int) -> int | None:
    """The most requests the draft forward pass fits in
    max_num_batched_tokens, which limits max_num_active_seqs, or None if
    unlimited (e.g. when it's sized by the target's scheduled tokens)."""
    return None

pcp_unsupported_reason(parallel_config)

Why prefill context parallelism can't be used, or None.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def pcp_unsupported_reason(self, parallel_config: "ParallelConfig") -> str | None:
    """Why prefill context parallelism can't be used, or None."""
    if self.supports_pcp:
        return None
    return (
        "MRV2 PCP only supports DSpark or single-module MTP speculative decoding."
    )

specialize(spec) classmethod

This config, or a more specific subclass for the settings.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

@classmethod
def specialize(cls, spec: "SpeculativeConfig") -> type["SpeculatorConfig"]:
    """This config, or a more specific subclass for the settings."""
    return cls

_get_draft_declared_config_class(spec_config)

The config declared by the draft model (see SpeculativeDraftModel).

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def _get_draft_declared_config_class(
    spec_config: "SpeculativeConfig",
) -> type[SpeculatorConfig] | None:
    """The config declared by the draft model (see `SpeculativeDraftModel`)."""
    draft = spec_config.draft_model_config
    qualname = getattr(draft, "speculator_config_class", None)
    return resolve_obj_by_qualname(qualname) if qualname is not None else None

_resolve_checkpoint_config_class(spec_config, configured_cls)

The built-in config for the draft checkpoint: detected if no method is configured, else the configured one, checked against the speculator the draft model declares (if any).

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def _resolve_checkpoint_config_class(
    spec_config: "SpeculativeConfig",
    configured_cls: type[DraftModelSpeculatorConfig],
) -> type[DraftModelSpeculatorConfig]:
    """The built-in config for the draft checkpoint: detected if no method is
    configured, else the configured one, checked against the speculator the
    draft model declares (if any).
    """
    declared_cls = _get_draft_declared_config_class(spec_config)
    if spec_config.method is not None:
        if declared_cls is not None and declared_cls.base_method != spec_config.method:
            draft = spec_config.draft_model_config
            declared = declared_cls.base_method
            requested = spec_config.custom_method or spec_config.method
            raise ValueError(
                f"speculative_config requested method={requested!r}, but "
                f"the draft model {draft.model!r} is a {declared!r} checkpoint "
                f"(model_type={getattr(draft.hf_config, 'model_type', None)!r}, "
                f"architectures={getattr(draft, 'architectures', None)}). Pass a "
                f"matching method or a matching draft checkpoint. Use "
                f"method={declared!r} if that is what you intended."
            )
        return configured_cls
    if spec_config.custom_method is not None:
        # Not based on a built-in speculator, so there's nothing to check.
        return configured_cls

    if declared_cls is not None:
        assert issubclass(declared_cls, DraftModelSpeculatorConfig)
        return declared_cls
    for method in _CHECKPOINT_DETECTION_ORDER:
        config_cls = resolve_obj_by_qualname(_CONFIG_BY_METHOD[method])
        if config_cls.matches(spec_config):
            return config_cls
    return configured_cls

get_custom_speculator_config_class(method)

The config of a registered custom speculative method, or None.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def get_custom_speculator_config_class(
    method: str | None,
) -> type[SpeculatorConfig] | None:
    """The config of a registered custom speculative method, or None."""
    if method not in _CUSTOM_CONFIGS:
        return None
    config_cls = _CUSTOM_CONFIGS[method]
    if isinstance(config_cls, str):
        config_cls = resolve_obj_by_qualname(config_cls)
    if not (isinstance(config_cls, type) and issubclass(config_cls, SpeculatorConfig)):
        raise TypeError(
            f"The config of speculative method '{method}' must be a subclass of "
            f"{SpeculatorConfig.__module__}.{SpeculatorConfig.__qualname__}."
        )
    if config_cls.speculator_class is None:
        raise ValueError(
            f"The config of speculative method '{method}' must set speculator_class."
        )
    return config_cls

get_method_config_class(spec_config)

The config class for spec_config.method, regardless of the draft checkpoint (for setting up spec_config).

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def get_method_config_class(spec_config: "SpeculativeConfig") -> type[SpeculatorConfig]:
    """The config class for `spec_config.method`, regardless of the draft
    checkpoint (for setting up `spec_config`)."""
    if spec_config.custom_method is not None:
        custom_cls = get_custom_speculator_config_class(spec_config.custom_method)
        assert custom_cls is not None
        return custom_cls
    method = spec_config.method
    if method is None:
        # Not configured: a draft model, unless the checkpoint shows otherwise.
        method = "draft_model"
    if method not in _CONFIG_BY_METHOD:
        raise ValueError(f"Unsupported speculative method: '{method}'")
    return resolve_obj_by_qualname(_CONFIG_BY_METHOD[method])

get_speculator_config_class(spec_config)

The config class for a set-up spec config.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def get_speculator_config_class(
    spec_config: "SpeculativeConfig",
) -> type[SpeculatorConfig]:
    """The config class for a set-up spec config."""
    config_cls = get_method_config_class(spec_config)
    if spec_config.custom_method is not None:
        return config_cls
    declared_cls = _get_draft_declared_config_class(spec_config)
    if declared_cls is not None and issubclass(declared_cls, config_cls):
        return declared_cls
    return config_cls.specialize(spec_config)

register_speculative_method(name, config_cls)

Register a custom speculative method, used with method=name.

config_cls is its SpeculatorConfig subclass, or its qualified name to import it lazily. Register it from a vLLM general plugin, so that it's registered in every process. This is an experimental interface that may change between releases.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def register_speculative_method(
    name: str, config_cls: "type[SpeculatorConfig] | str"
) -> None:
    """Register a custom speculative method, used with `method=name`.

    `config_cls` is its `SpeculatorConfig` subclass, or its qualified name to
    import it lazily. Register it from a vLLM general plugin, so that it's
    registered in every process. This is an experimental interface that may
    change between releases.
    """
    from vllm.config.speculative import SpeculativeMethod

    if name in get_args(SpeculativeMethod) or name in _CONFIG_BY_METHOD:
        raise ValueError(f"'{name}' is a built-in speculative method.")
    if name in _CUSTOM_CONFIGS and _CUSTOM_CONFIGS[name] != config_cls:
        raise ValueError(f"Speculative method '{name}' is already registered.")
    _CUSTOM_CONFIGS[name] = config_cls

registered_speculative_methods()

The names of the registered custom speculative methods.

Source code in vllm/v1/worker/gpu/spec_decode/speculator_config.py

def registered_speculative_methods() -> list[str]:
    """The names of the registered custom speculative methods."""
    return sorted(_CUSTOM_CONFIGS)