o )i@sRUddlZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZmZmZmZmZmZddlmZmZddlmZddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ddl%Z&ddl'Z'ddl(m)Z)m*Z*m+Z+m,Z,dd l-m.Z.dd l/m0Z1dd l2m3Z3m4Z4m5Z5ddl6m7Z7dd l8m9Z9dd l:m;Z;mZ>m?Z?ddl@mAZAmBZBmCZCmDZDddlEmFZFmGZGddlHmIZImJZJddlKmLZLmMZMddlNmOZOddlPmQZQddlRmSZSddlTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZaddlbmcZcddldmeZemfZfddlgmhZhmiZimjZjmkZkmlZlerkddlmmnZnddlompZpddlPmqmrmsZtddlumqmvZwddlPmQZQddlxmyZyddlzm{Z{ddl|m}Z}ddl~mZe"eeegeffZn-eZneZpeZyeZQeZeZ{eZ}eZe"eeefeegeffZejded Ztejded!ZweOeZe!d"end#Zed$Zed%Zed&Zed'Zed(Zed)Zd*d+ggd,d-gd'Zeeeefed.<ggd/gd'Zeeeefed0<gd1Zeeeeeeffed2<d3d4Zddd5d6ed7eed8eed9eeeeeefffd:d;Ze5Gdd?d?e ZGd@dAdAeejZdBeed9eeeffdCdDZdBeLdEed9efdFdGZdBeLdEed9efdHdIZedJZedKZedLZeMe.e)dMdNdOGdPdQdQZeMe.GdRdSdSZedTZeMe.e)dMdNdOGdUdVdVZedWZeMe.GdXdYdYZedZZeMe.e)dMdNdOGd[d\d\ZeMe.Gd]d^d^ZeMe.Gd_d`d`Ze'je'je'je'je'jdaZdbdbdbdbdcZddedee'jfdfdgZddedee'jfdhdiZdjedkepdleefdmdnZddedoe'jdpefdqdrZddsdjedkepdee"ee'jfdpedleed9e'jf dtduZ  ddvepdweedxeedyedzeed{eed|eed9efd}d~Zdedee"eeeffddZedZeMe.GdddZedZeMe.GdddZedZedZeeefZeMe.GdddZeMe.GdddZeMe.e)dMdNdOGdddZdaeeed<daeeed<e ddedeefddZeddddZd9efddZd9efddZddZddZe!dZ ddedeedeeed9eeeffddZeMe.GdddZdkedeeefd9efddZdS)N)Mapping)contextmanager)MISSINGFieldfieldfields is_dataclassreplace)cached_property lru_cache) find_spec) TYPE_CHECKINGAnyCallableClassVarLiteralOptionalProtocolTypeVarUnioncastget_args) ConfigDictSkipValidationfield_validatormodel_validator) dataclass)_TYPES)Self assert_neverruntime_checkable)version) BlockSize CacheConfig CacheDType MambaDTypePrefixCachingHashAlgo)CompilationConfigCompilationLevel CUDAGraphMode PassConfig)DistributedExecutorBackendParallelConfig)SchedulerConfigSchedulerPolicy) ConfigTypeconfig) init_logger)QuantizationMethodscurrent_platform) ConfigFormat get_configget_hf_image_processor_configget_hf_text_configget_pooling_config)get_sentence_transformer_tokenizer_configis_encoder_decoderis_interleaved,maybe_override_with_speculators_target_modeltry_get_generation_configtry_get_safetensors_metadatatry_get_tokenizer_config uses_mrope)S3Model)is_s3maybe_model_redirect)DEFAULT_MAX_NUM_BATCHED_TOKENSLayerBlockType LazyLoadercommon_broadcastable_dtype random_uuid)DataclassInstance)PretrainedConfig)QuantizationConfig) LoadFormats)TensorizerConfig)LogitsProcessormodel_executorz'vllm.model_executor.layers.quantizationzvllm.model_executor.modelsDataclassInstanceT)bound) autogenerate embeddingembedclassifyscorereward transcriptiondraft)rTrZencoderVrWrYr[)rSrTpoolingr[)rTr]r[)rSnonerVrWrY)r^rVrWrYrTrZ)rUrVrWrXrYr[ _RUNNER_TASKSrVrWrY_RUNNER_CONVERTS))Z ForCausalLMrTr^)ZForConditionalGenerationrb)Z ChatModelrb)Z LMHeadModelrb)ZForTextEncodingr]rV)ZEmbeddingModelrc)ZForSequenceClassificationr]rW)ZForAudioClassificationrd)ZForImageClassificationrd)ZForVideoClassificationrd)ZClassificationModelrd)ZForRewardModelingr]rY)Z RewardModelre)ZModelrc_SUFFIX_TO_DEFAULTSccstEdHdSN)rfrhrh`/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/config/__init__.pyiter_architecture_defaultssrj) runner_type convert_type architecturerkrlreturncCsRtD]#\}\}}|dus||kr&|dus||kr&||r&|||ffSqdSrg)rjendswith)rmrkrlsuffixZdefault_runner_typeZdefault_convert_typerhrhritry_match_architecture_defaultss rqc@seZdZdefddZdS) SupportsHashrncCdSrgrhselfrhrhri compute_hashzSupportsHash.compute_hashN)__name__ __module__ __qualname__strrvrhrhrhrirrsrrc@s"eZdZdeeeffddZdS)SupportsMetricsInforncCrsrgrhrtrhrhri metrics_inforwz SupportsMetricsInfo.metrics_infoN)rxryrzdictr{r}rhrhrhrir|sr|c@seZdZdZdZdZdS) ModelImplrSvllmZ transformersN)rxryrzAUTOZVLLMZ TRANSFORMERSrhrhrhrirsrclsc Csdd}ttt|jd}t|tjst di}||jD]G\}}t|tj tj frDt|tj rDt|j tjrDt|j j tsEq#t|j j }t|tj rU|jn|jg}|D]}t|tjsdq[|||j<q[q#|S)zw Get any docstrings placed after attribute assignments in a class body. https://davidism.com/mit-license/ css0t|}t|d}|D] }||fV|}q dS)z Manually implement https://docs.python.org/3/library/itertools.html#itertools.pairwise Can be removed when Python 3.9 support is dropped. N)iternext)iterableiteratorabrhrhripairwises  zget_attr_docs..pairwiserzGiven object was not a class.)astparsetextwrapdedentinspect getsourcebody isinstanceClassDef TypeErrorAssign AnnAssignExprvalueConstantr{cleandoctargetstargetNameid) rrZcls_nodeoutrrdocrrrhrhri get_attr_docss,      rnamecCst|stdddt|D}||vr!td|d|jd||}|j}tur1t|dS|j}tur=t|dSt|jd|d ) zrGet the default factory field of a dataclass by name. Used for getting default factory fields in `EngineArgs`.z#The given class is not a dataclass.cSsi|]}|j|qSrhr.0frhrhri zget_field..zField 'z' not found in .default_factory)defaultz. must have a default value or default factory.) rrr ValueErrorrxrrrr)rr cls_fieldsZ named_fieldrrrhrhri get_fields  rcstfddt|DjS)Nc3s|] }|jkr|VqdSrgrrrrhri sz is_init_field..)rrinit)rrrhrri is_init_fieldsr)rSZslowZmistralZcustom)rShalffloat16bfloat16floatfloat32) raw_logprobsZ raw_logitsZprocessed_logprobsZprocessed_logitsT)Zarbitrary_types_allowedr0c @sBeZdZUdZdZeed< dZeed< dZ e ed< dZ e e ed< dZeeed < dZeed < d Zeed < dZeeejfed < dZe eed< dZe eed< dZeed< dZe eed< dZe eed< eedZ eee!fed< dZ"e e#ed< dZ$e eed< dZ%eeed< dZ&e eed< dZ'ee e(ed< d Z)eed< dZ*eed< dZ+eed< d Z,e-ed!< d Z.eed"< d Z/eed#< d Z0eed$< d Z1eed%< dZ2e eee3efed&< eedZ4eeefed'< d Z5eed(< d Z6eed)< eedZ7eeeee!ffed*< d+Z8eed,< e9j:j;Ze?ed/< dZ@e eee!fed0< d1ZAeed2< eedZBeee!fed3< ed d4ZCe d5ed6< dZDe eed5fed7< dZEe eed8< dZFeed9< eedZGeee!fed:< d ZHeed;< eIj:j;ZJeeeIfed<< dZKe eed=< dZLe e3eeeMeNfed>< d?efd@dAZOddBdCZPeQddDdEeRdFe!d?e!fdGdHZSeTdIdEddKdLZUd?efdMdNZVd?efdOdPZWeXdQdRZYeXd?e3efdSdTZZeXd?efdUdVZ[ded ed?dfdWdXZ\d?e dYfdZd[Z]dFed?dfd\d]Z^d^d_Z_d?e d5fd`daZ`ddbdcZadde3ed?ebfdedfZcdde3eded?ebfdgdhZddde3ediebd?eefdjdkZfdde3ediebde d?eefdldmZgdde3edneed?e3ehfdodpZidde3ed?ejdqfdrdsZkdde3edneed?e3ehfdtduZldde3ediebdneed?e3ehfdvdwZmdxdyZnddzd{Zodd|d}Zpdd~dZqdddZr ? dddZs ? dddZt ? dddZud?e efddZvd?efddZwd?efddZxeXd?efddZyd?efddZzd?efddZ{ddd?efddZ|ddd?efddZ}ddd?e~eeffddZddd?efddZejfddded?efddZd?e efddZdddZd?eee!ffddZd?eee!ffddZeXd?efddZeXd?efddZeXd?efddZeXd?efddZeXd?efddZd?efddZeXd?efddZeXd?efddZeXd?efddZeXd?efddZeXd?efddZeXd?efdd„ZeXddĄZeXd?efddƄZeXd?efddȄZeXd?efddʄZeXdd̄ZeXd?efdd΄ZdefddЄZdS) ModelConfigzConfiguration for the model.zQwen/Qwen3-0.6BmodelrSrunnerconvertNtask tokenizertokenizer_modeFtrust_remote_codedtypeseedhf_config_pathallowed_local_media_pathrevision code_revisionr rope_scaling rope_thetatokenizer_revision max_model_lenspec_target_max_model_len quantization enforce_eageri max_seq_len_to_capture max_logprobsr logprobs_modedisable_sliding_windowdisable_cascade_attnskip_tokenizer_initenable_prompt_embedsserved_model_namelimit_mm_per_promptinterleave_mm_stringsskip_mm_profilingmedia_io_kwargsTuse_async_output_proc config_formathf_token hf_overridesmm_processor_kwargsmm_processor_cache_gboverride_neuron_configr PoolerConfig pooler_configoverride_pooler_configlogits_processor_patterngeneration_configoverride_generation_configenable_sleep_mode model_imploverride_attention_dtypelogits_processorsrncCsg}||j||j||j||j||j||j||j||j||j ||j ||j ||j ||j ||j||jt|}t|tt|S) WARNING: Whenever a new field is added to this config, ensure that it is included in the factors list if it affects the computation graph. Provide a hash that uniquely identifies all the configs that affect the structure of the computation graph from input ids/embeddings to the final hidden states, excluding anything before input ids/embeddings and after the final hidden states. )appendrrrrrrrrrrrrrr hf_configZto_json_stringr{assert_hashablehashlibsha256r\ hexdigest)rufactorsZ str_factorsrhrhrirvs&              zModelConfig.compute_hashc sNtjrjdurd_tjstdjjdkr*tjj j j d\__ t jj _ tj_j durAj_ jdurJj _tj _ tjtr\tj_tjrgi}j}nj}d}jrdji}||t|}d|d}tjt|dd jdurd ji}||t|}d |d}tjt|dd jj tj}r|d krtd durt dddl!m"}j#dur|$stjddd j%r|&st dtj'trt(j'_'t)jpjj j j*j'||d}|_+t,j+_-t.j-dd_/0_1t2jj3j d_4j5j6} | 7} | 8} dt9dt:ffdd } j;durd} d}d}d}j;t|d%|}tj|tdd ?j_@Aj@j>_Bj@dkr| stCd}jB|vrt d&j@dkr| stCd}jB|vrd'd(D|d)}t d*|d+Ej@jB_F| G\}}|_H|_ItJd,|K_LtMjj+jNj@dkj d-_NjOsitPj-ritjsitj}d.vritQd/j-jR|j-jSd0_OjT_UVjT_TW_XjOrdj-_SjYsZ|[sj\rt d1d2_]^_`dS)3NrzThe global random seed is set to %d. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.r[)rrrrrz[`--rope-scaling` will be removed in a future release. 'Please instead use `--hf-overrides 'z'`) stacklevelrzY`--rope-theta` will be removed in a future release. 'Please instead use `--hf-overrides ' FLASHINFERZ flashinferzVLLM_ATTENTION_BACKEND is set to FLASHINFER, but flashinfer module was not found. See https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile for instructions on how to install it.r3z;override-attention-dtype is set but not using ROCm platformz0Sleep mode is not supported on current platform.)hf_overrides_kwhf_overrides_fnattention_chunk_size)rrrrncsR|dks|dkr dS|dkrdS|dkrdS|dkr'}|dkr%dSdSdS)NrUrVrWrYrXr^)_get_default_pooling_task)rnew_task architecturesrurhri_task_to_convertms z3ModelConfig.__post_init__.._task_to_convertrSzdThe 'task' option has been deprecated and will be removed in v0.13.0 or v1.0, whichever comes first.zPlease remove this option.rTr]zgPlease replace this option with `--runner generate` to continue using this model as a generative model.zcPlease replace this option with `--runner pooling` to continue using this model as a pooling model.zPlease remove this optionz+Please replace this option with `--convert z2` to continue using this model as a pooling model.zFThe model should be a generative or pooling model when task is set to r z0This model does not support `--runner generate`.<|>zHThis model does not support `--runner pooling`. You can pass `--convert z" to adapt it into a pooling model.zResolved architecture: %s)is_pooling_modelr)ZXFORMERSrz%s has interleaved attention, which is currently not supported by the %s backend. Disabling sliding window and capping the max length to the sliding window size (%d).Tz5`override_neuron_config` is only supported on Neuron.F)aenvs VLLM_USE_V1rZVLLM_ENABLE_V1_MULTIPROCESSINGloggerwarningrr=rrrrget_served_model_namerrDrrrr{callablerrupdatejsondumpswarningswarnDeprecationWarningr!maybe_pull_model_tokenizer_for_s3ZVLLM_ATTENTION_BACKENDr rvllm.platformsr4ris_rocmrZis_sleep_mode_availablerr5r6rrr8hf_text_configgetattrr_get_encoder_configencoder_configr7rZhf_image_processor_configrregistryis_text_generation_modelr TaskOption ConvertTyperr_AssertionErrorr_get_runner_typerk_get_convert_typerlrajoin_get_supported_taskssupported_tasksZinspect_model_cls _model_info _architectureinfo_init_pooler_configr_get_and_verify_dtyperrr< warning_once model_typesliding_windowrZoriginal_max_model_lenget_and_verify_max_len_init_multimodal_configmultimodal_configr_verify_tokenizer_mode is_neuronrconfig_updated_verify_quantization_verify_cuda_graph_verify_bnb_config)rurrZ hf_overrideZhf_overrides_strmsgbackendr4rrZis_generative_modelrrrrZ msg_prefixZmsg_hintZis_generative_taskZis_pooling_taskZgenerate_convertsZpooling_convertsZconvert_optionZ model_infoarchrhrri __post_init__s                                        zModelConfig.__post_init__beforemodercCst|tr |S|Srg)rr{lower)rrrhrhrivalidate_quantization_befores z(ModelConfig.validate_quantization_beforeafterrucCs,t|jts tdt|jtstd|S)Nz/tokenizer must be a string after __post_init__.z5max_model_len must be an integer after __post_init__.)rrr{rrintrtrhrhrivalidate_model_config_afters  z'ModelConfig.validate_model_config_aftercCs*t|d|jdkr dS|j|jkrdSdS)zsDetermine which Transformers backend class will be used if `model_impl` is set to `transformers` or `auto`.rkr]ZTransformersModelZTransformersForMultimodalLMZTransformersForCausalLM)rrrrrtrhrhri_get_transformers_backend_cls s  z)ModelConfig._get_transformers_backend_clscCs|j|kS)z;Check if the model is using the Transformers backend class.)rmrArtrhrhriusing_transformers_backendz&ModelConfig.using_transformers_backendcCstjSrg) me_modelsZ ModelRegistryrtrhrhrirszModelConfig.registrycCst|jdgS)NrrrrtrhrhrirrCzModelConfig.architecturescC|jS)z$The architecture vllm actually used.)r%rtrhrhrirm!szModelConfig.architecturecCst|s t|s dSt|r4t}|j|gdd||_|j|_||kr4|j|gdd|j|_dSt|rJt}|j|gdd|j|_dSdS)zPull model/tokenizer from S3 to temporary directory when needed. Args: model: Model name or path tokenizer: Tokenizer name or path N)z*.modelz*.pyz*.json)Z allow_pattern)z*.ptz *.safetensorsz*.binz *.tensors)Zignore_pattern)rCrBZ pull_filesZ model_weightsdirrr)rurrZs3_modelZ s3_tokenizerrhrhrir&s. z-ModelConfig.maybe_pull_model_tokenizer_for_s3MultiModalConfigcCs,|jjrt|j|j|j|j|j|jdSdS)N)limit_per_promptrrrrr) r$Zsupports_multimodalrHrrrrrrrtrhrhrir-Jsz#ModelConfig._init_multimodal_configcCs|}||_||_dSrg)get_multimodal_configr)rur mm_configrhrhriset_mm_processor_cache_gbVs z%ModelConfig.set_mm_processor_cache_gbcCst|j|jSrg)r:rrrtrhrhrir\szModelConfig._get_encoder_configcCs|jdkrIt|jtrtdi|j|_|jpt}t|j|j}|dur;|D]\}}t ||dur:t |||q)|j j }|j durG||_ |SdS)Nr]rh)rkrrr~rr9rritemsrsetattrr$default_pooling_type pooling_type)rurZ base_configkvrOrhrhrir'`s"     zModelConfig._init_pooler_configcCsBtt|j}|ttvrtd|jdttd||_dS)NzUnknown tokenizer mode: . Must be one of r)r TokenizerModerr<rr)rurrhrhrir/ws   z"ModelConfig._verify_tokenizer_modercCsv|j}t|j|jr dS|D]*}||vr(|||rdS|||r(dSt|}|r8|\}\}}|SqdS)Nr]rT)rr9rrget_supported_archsrrrq)rurrr7match_rkrhrhri_get_default_runner_types    z$ModelConfig._get_default_runner_typecCs.|dkr|S||}|dkrtd||S)NrSrTz]Resolved `--runner auto` to `--runner %s`. Pass the value explicitly to silence this message.)rXr r&)rurrrkrhrhrirs zModelConfig._get_runner_typerkcCs|j}|D]4}||vr'|dkr|||rdS|dkr'|||r'dSt||d}|r9|\}\}}|Sq|dkr@dSdS)NrTr^r]rkrV)rrUrrrq)rurrkrr7rVrWrlrhrhri_get_default_convert_types.   z%ModelConfig._get_default_convert_typecCs0|dkr|S|||}|dkrtd||S)NrSr^z_Resolved `--convert auto` to `--convert %s`. Pass the value explicitly to silence this message.)rZr r&)rurrkrrlrhrhrir szModelConfig._get_convert_typerlcCs^|j}|||r dgStt}|||s|tdvr"|d|||r-|d|S)NrZrT)rZis_transcription_only_modellist _ResolvedTaskrrarZis_transcription_model)rurrlrr#rhrhri_get_supported_generation_taskss       z+ModelConfig._get_supported_generation_tasksr`cCsP|j||r dS|D]}t|dd}|r%|\}\}}|dks!J|Sq dS)NrWr]rYr^rV)rZis_cross_encoder_modelrq)rurr7rVrWrlrhrhrirs  z%ModelConfig._get_default_pooling_taskcCsV|j}tt}|||s|tdvr)|d|dkr"||n|}|||S)Nr]r\r^)rr[r\rrarr)rurrlrr#Z extra_taskrhrhri_get_supported_pooling_taskss      z(ModelConfig._get_supported_pooling_taskscCsB|dkr |||S|dkr|||S|dkrdgSt|dS)NrTr]r[)r]r^r)rurrkrlrhrhrir"s z ModelConfig._get_supported_taskscCst|jdd}|durt|jdd}|S|did}|dkrI|did}|dkr4d|d <|S|d kr>d |d <|S|durItd ||S) Nquantization_configZcompression_configZproducerrmodeloptr quant_algoZFP8 quant_methodZNVFP4 modelopt_fp4zUnknown ModelOpt quant algo: )rrgetr)ru quant_cfgZ producer_namerarhrhri_parse_quant_hf_config(s,z"ModelConfig._parse_quant_hf_configc sxtj}gd}|jdurttj|j|_|}|dur|dd}|dd}||d<gdfdd|D}|}|D],}t |}| ||j}|durm|t tjvrf|vrft d |d |}||_nqA|jdurw||_n|j|krt d |d |jd |jdur|j|vrt d|jd|dddl m} | |j|j|vrtd|jdSdSdS)N)Zfp8marlinr`gptq_marlin_24 gptq_marlin awq_marlinZ fbgemm_fp8compressed-tensorsZ experts_int8Zquarkrcbitblas gptq_bitblasincrbrZcompressed_tensorsrk) rgrlrhrirmrjZipexZ moe_wna16r`rccsg|]}|vr|qSrhrh)rq overridesrhri esz4ModelConfig._verify_quantization..zQuantization method z is an override but is has not been added to the `overrides` list above. This is necessary to ensure that the overrides are checked in order of preference.z3Quantization method specified in the model config (zS) does not match the quantization method specified in the `quantization` argument ().zUnknown quantization method: rSrrr3z^%s quantization is not fully optimized yet. The speed can be slower than non-quantized models.)me_quantZQUANTIZATION_METHODSrrr2rfrdr<r get_quantization_configZoverride_quantization_methodrrrr4Zverify_quantizationr r ) ruZsupported_quantizationZoptimized_quantization_methodsrerbZquantization_methodsrmethodZquantization_overrider4rhrprir2>sv            z ModelConfig._verify_quantizationcCs~|j}|jrt|t|jdd}t|j||_dg}|jj|vp"|j}|r9|js;t r=t d|jjd|_dSdSdSdS)NZmax_source_positionsrmllamazGCUDA graph is not supported for %s on ROCm yet, fallback to eager mode.T) rr;maxrrminrr*rr4rr r )ruZeffective_max_seq_lenZROCM_UNSUPPORTED_MODELSZunsupported_rocmrhrhrir3s0    zModelConfig._verify_cuda_graphcCs`|jdk}t|jdddu}|r|jjddnd}t||||j gr.tdd|_dSdS)z The current version of bitsandbytes (0.46.1) with 8-bit models does not yet support CUDA graph. # TODO Remove this when bitsandbytes supports. Z bitsandbytesr_NZ load_in_8bitFzQCUDA graph is not supported on BitsAndBytes 8bit yet, fallback to the eager mode.T) rrrr_rdallrr r )ruZis_bitsandbytesZhas_quantization_configZis_8bitrhrhrir4s.   zModelConfig._verify_bnb_configcCsDgd}d}|D]}t|j|d}|dkrnq|dkr tddS)N)Zmoe_num_experts num_expertsZn_routed_expertsZnum_local_expertsrzYNumber of experts in the model must be greater than 0 when expert parallelism is enabled.)rrr)ruZnum_expert_namesr{rrhrhri_verify_with_expert_parallelismsz+ModelConfig._verify_with_expert_parallelism load_config LoadConfigcCsZt|jdr'ddlm}|||}|r)||jjd<d|jjvr+d|jjd<dSdSdSdS)Ndual_chunk_attention_configr)get_sparse_attention_configZsparse_attention_configZsparse_attention_enabledT)hasattrr-vllm.model_executor.model_loader.weight_utilsrr)rur~rZsparse_attn_configrhrhri"verify_dual_chunk_attention_configs$   z.ModelConfig.verify_dual_chunk_attention_configcCsr|jsdS|jdkrd|_dSddlm}||js d|_dStjr(d|_dS|jdkr0d|_|r7d|_dSdS)Nr|Frr3r]) rpipeline_parallel_sizerr4Zis_async_output_supportedrrZVLLM_USE_RAY_SPMD_WORKERrk)ruparallel_configspeculative_config device_configr4rhrhriverify_async_output_procs"     z$ModelConfig.verify_async_output_procrr,cCs|jdkr|jdusJdt|jdd}|j}||dkr)td|d|d|jr0||j}|dkrK|j |j |sCt d |j rMd |_ dSdSdS) NZexternal_launcherzpSeed must be set when using external launcher backend to make sure sampling results are the same across workers.num_attention_headsrz!Total number of attention heads (z-) must be divisible by tensor parallel size (rsr|zlPipeline parallelism is not supported for this model. Supported models implement the `SupportsPP` interface.F)distributed_executor_backendrrrtensor_parallel_sizerZenable_expert_parallelr}rrZis_pp_supported_modelrNotImplementedErrorr)rurZtotal_num_attention_headsrrrhrhriverify_with_parallel_config s8     z'ModelConfig.verify_with_parallel_configcCt|jddS)z?Get the sliding window size from the HF text config if present.r+Nrrrtrhrhriget_sliding_window,rCzModelConfig.get_sliding_windowcCr)NZ vocab_sizerrrtrhrhriget_vocab_size0zModelConfig.get_vocab_sizecCr)N hidden_sizerrrtrhrhriget_hidden_size3rzModelConfig.get_hidden_sizecCsRt|jdsdS|jjdvr|jjduS|jjdkr'|jjjdvo&|jjduSdS)Nr*F) deepseek_v2 deepseek_v3 deepseek_mtpZkimi_k2eagle)rr)rrr* kv_lora_rankrrtrhrhriis_deepseek_mla6s    zModelConfig.is_deepseek_mlacCs|jr"t|jdd}|jr|jj|St|jdd}|r"|r"||St|jdr2|jjdkr2|jjS|jr7dSt|jdddurD|jj S|jj |jj S)Nqk_rope_head_dimrqk_nope_head_dimr*zamba2head_dim) rrruse_mlarrr*Zattention_head_dimis_attention_freerrr)rurrrhrhri get_head_sizeEs2 zModelConfig.get_head_sizecCsgd}|jj|vot|jdd}|st|jddrdS|jjdkr2d|jjvr.|jjdS|jjS|jjdkrBt|jjd|jjS|jjd kra|jjD]}|jjs\|jj|jj SqLt d |j rfd Sgd }|D]}t|j|d }|d ur}|Sql|jjS)z%Returns the total number of KV heads.)ZfalconZ RefinedWebZRefinedWebModelZnew_decoder_architectureFZ multi_queryr|ZmptZ kv_n_headsZdbrxz nemotron-nasz%Couldn't determine number of kv headsr)Z n_head_kv num_kv_headsZnum_key_value_headsZmulti_query_group_numN) rr*rrZ attn_configr block_configs attentionno_opZn_heads_in_group RuntimeErrorr)ruZfalcon_model_typesZnew_decoder_arch_falconblock attributesattrrrhrhriget_total_num_kv_headsbsB           z"ModelConfig.get_total_num_kv_headscCs"|jrdS|}td||jS)z'Returns the number of KV heads per GPU.r|)rrrxr)rurZtotal_num_kv_headsrhrhriget_num_kv_headss zModelConfig.get_num_kv_headscCst|jdd}||jS)Nrr)rrr)rurZ num_headsrhrhriget_num_attention_headss z#ModelConfig.get_num_attention_headscCs~ddlm}|jjdks|jjdks|jjdkr t|jdd}nt|jdd}|j|j|j}|j}||||\}}||fS)Nr)get_pp_indicesrmimo_mtp glm4_moe_mtpnum_nextn_predict_layersnum_hidden_layers) Zvllm.distributed.utilsrrr*rrZrankrr)rurrZtotal_num_hidden_layersZpp_rankZpp_sizestartendrhrhriget_layers_start_end_indicess     z(ModelConfig.get_layers_start_end_indicescCs||\}}||Srg)r)rurrrrhrhriget_num_layersszModelConfig.get_num_layers block_typec s^tjk}|j o|j o|j }||\}}|r"|r ||SdS|jr-|r)dS||S|jrA|jj}tdd|||DSt |jdd}|dur{t |j drl|j j dkrl|rgtdd|||DS| |Stfdd|||DSt |jd d} | rtd d| ||DS|dur| durtd jd td d| ||DS)Nrcss|]}|jj VqdSrg)rr)rbcrhrhrirz;ModelConfig.get_num_layers_by_block_type..Zlayers_block_typer*rcs|]}|dkVqdS)ZhybridNrhrtrhrhrirsc3s|]}|jkVqdSrg)rrrrhrirrattn_type_listcsrr|NrhrrhrhrirztThe model is an hybrid without alayers_block_type or an attn_type_list in the hf_config,cannot determine the num of z layerscsrrrhrrhrhrirr)rFr is_hybrid has_noopsrrrrsumrrrr*rrr) rurrZattn_block_typeZis_transformerrrrZlayers_block_type_valuerrhrriget_num_layers_by_block_typesX      z(ModelConfig.get_num_layers_by_block_typecCs(t|jdd}|durt|jdd}|S)z; Returns the mamba chunk size if it exists Zmamba_chunk_sizeN chunk_sizer)rurrhrhriget_mamba_chunk_sizesz ModelConfig.get_mamba_chunk_sizecCs|jdur td|jS)z Get the multimodal configuration of the model. Raises: ValueError: If the model is not multimodal. NzThe model is not multimodal.)r.rrtrhrhrirJs z!ModelConfig.get_multimodal_configcCsJ|jdvrt|jp |j|j|jd}nt|j|jd}|dur!iS|S)a This method attempts to retrieve the non-default values of the generation config for this model. The generation config can contain information about special tokens, as well as sampling parameters. Which is why this method exists separately to `get_diff_sampling_param`. Returns: A dictionary containing the non-default generation config. >rSrrr)rN)rr>rrrrZ to_diff_dict)rur0rhrhrir>s  z%ModelConfig.try_get_generation_configcs|jdkrin||jgd}tfdd|Dr6fdd|D}d|vr5|d|d<ni}|r?td |S) a This method returns a dictionary containing the non-default sampling parameters with `override_generation_config` applied. The default sampling parameters are: - vLLM's neutral defaults if `self.generation_config="vllm"` - the model's defaults if `self.generation_config="auto"` - as defined in `generation_config.json` if `self.generation_config="path/to/generation_config/dir"` Returns: A dictionary containing the non-default sampling parameters. r)Zrepetition_penaltyZ temperatureZtop_kZtop_pZmin_pmax_new_tokensc3s|]}|vVqdSrgrhrprrhrirKrz6ModelConfig.get_diff_sampling_param..cs&i|]}|dur||qSrg)rdrrrhrirLs z7ModelConfig.get_diff_sampling_param..rZ max_tokenszDefault sampling parameters have been overridden by the model's Hugging Face generation config recommended from the model creator. If this is not intended, please relaunch vLLM instance with `--generation-config vllm`.)rr>r ranypopr r))ruZavailable_paramsZdiff_sampling_paramrhrriget_diff_sampling_param,s(   z#ModelConfig.get_diff_sampling_paramcCs$ tr |jjdkr dSt|jS)z*Extract the HF encoder/decoder model flag.rwF)r4r0rr*r;rtrhrhrir;`s   zModelConfig.is_encoder_decodercCs t|jSrg)rArrtrhrhrirAo zModelConfig.uses_mropecC |jduSrg)r.rtrhrhriis_multimodal_modelsrzModelConfig.is_multimodal_modelcC|j}|dur dS|jdkS)z7Whether the multi-modal processor should output hashes.NFrr.rrurKrhrhriprocessor_return_mm_hashesw z&ModelConfig.processor_return_mm_hashescCr)z:Whether the multi-modal processor cache should be enabled.NFrrrrhrhrienable_mm_processor_cacherz%ModelConfig.enable_mm_processor_cachecCs|j}|dur dStjSNr)r.rZVLLM_MM_INPUT_CACHE_GIBrrhrhriget_mm_input_cache_gbsz!ModelConfig.get_mm_input_cache_gbcCs|jjp|jdkS)NrW)r$Zsupports_cross_encodingrlrtrhrhriis_cross_encoderszModelConfig.is_cross_encodercC|jjSrg)r$Z supports_pprtrhrhriis_pp_supportedzModelConfig.is_pp_supportedcCrrg)r$Zsupports_multimodal_raw_inputrtrhrhri!is_multimodal_raw_input_supportedrz-ModelConfig.is_multimodal_raw_input_supportedcCrrg)r$rrtrhrhrirrzModelConfig.is_attention_freecCrrg)r$rrtrhrhrirrzModelConfig.is_hybridcCrrg)r$rrtrhrhrirrzModelConfig.has_noopscCrrg)r$has_inner_statertrhrhrirrzModelConfig.has_inner_statecCs |jj Srg)r$Zsupports_v0_onlyrtrhrhriis_v1_compatiblerzModelConfig.is_v1_compatiblecCs|jotj Srg)rrZVLLM_MLA_DISABLErtrhrhrirrCzModelConfig.use_mlacCs tt|jddpt|jddS)Nmatryoshka_dimensions is_matryoshkaF)boolrrrtrhrhrirs zModelConfig.is_matryoshkacCst|jddS)NrrErtrhrhrirrCz!ModelConfig.matryoshka_dimensionscCr)N use_pad_tokenTrErtrhrhrirszModelConfig.use_pad_tokenc Csfd}|jdkrt|jdddkrt|j|j|jd}t|j|||j | |j |j d}t d||S)Nr]Zposition_embedding_typerabsoluter)rtokenizer_configrrr+rrzUsing max model len %s)rkrrr@rrr_get_and_verify_max_lenrrrrrr r&)rurrrhrhrir,s,  z"ModelConfig.get_and_verify_max_lenrnN)rurrnr)r~rrnN)rr,rnN)rnrH)rxryrz__doc__rr{__annotations__r RunnerOptionr ConvertOptionrrrrrrrTrrrr ModelDTypetorchrr?rrrrrr~rrrrrrrrr2rrrr LogprobsModerrrrrr[rrrrrr5rrrrr HfOverridesrrrrrrrrrrrrrtyperOrvr8r classmethodr=rr@rArBpropertyrrrmrr-rLrr'r/ RunnerTyperXrrrZr r\r]rrr^r"rfr2r3r4r}rrrrrrrrrrrtuplerrrFrrrrJr>rr;rArrrrrrrrrrrrrrrrr,rhrhrhrirs                 "      !     $            Z     !6    7  4  rc@seZdZUdZdZeeefed< dZ e eed< e e dZ ee efed< dZe eed< dZe eeeefed < d Zeed < d Zeee eeffed < defddZddZdS)rz,Configuration for loading the model weights.rS load_formatN download_dirrmodel_loader_extra_configdeviceignore_patternsTuse_tqdm_on_loadcpupt_load_map_locationrncC"g}tjt|dd}|SrFusedforsecurityrmd5r{r\rrurhash_strrhrhrirvzLoadConfig.compute_hashcCsB|j|_|jdurt|jdkrtd|jdSdg|_dS)Nrznum_speculative_tokens was provided without speculative model.r|zprompt_lookup_min=z must be > 0zprompt_lookup_max=z must be <= prompt_lookup_max=rrrr[rrrrrrrrrrrrrrr)rrzeagle-zeagle3-rrr)rrrrzeAll Deepseek MTP models only have one layer. Might need some code changes to support multiple layers.rzSpeculative decoding with draft model is not supported yet. Please consider using other speculative decoding methods such as ngram, medusa, eagle, or deepseek_mtp.z;Chunked prefill and EAGLE are not compatible when using V0.)SpeculatorsConfig) EAGLEConfig)rvr*num_lookahead_tokensr znum_speculative_tokens:z must be divisible by n_predict=cSsg|]}|ddqS)r|rrhrirhrhrirrs z3SpeculativeConfig.__post_init__..cSs t||fSrg)r)rrhrhris z1SpeculativeConfig.__post_init__..)keyrh)7rrrrr*rvrrrrrrrrrrrrrrrrrrrrrrr!r<rr r rrrrZvllm.transformers_utils.configsr#Z%vllm.transformers_utils.configs.eagler$rrr%rrr{ranger literal_evalsorted_verify_and_get_draft_tpr#_maybe_override_draft_max_model_lencreate_draft_parallel_config)rur#r$Z eagle_configr Z tree_choicesrhrhrir8s^                                   zSpeculativeConfig.__post_init__speculative_max_model_lendraft_max_model_lentarget_max_model_lencCsN|dur"||krtd|d|||kr td|d||St||S)aDetermine the max sequence len for the draft model. This is usually the draft_max_model_len, but may be the target_max_model_len if it is less than the draft_max_model_len, or may be speculative_max_model_len if it is specified. This is necessary so that sequences do not exceed the capacity of the draft model or the target model. speculative_max_model_len is mainly used for testing that sequences can skip speculation. Nzspeculative_max_model_len=z+ cannot be larger than draft_max_model_len=z, cannot be larger than target_max_model_len=)rry)r1r2r3rhrhrir/s  z5SpeculativeConfig._maybe_override_draft_max_model_len&speculative_draft_tensor_parallel_sizedraft_hf_configcCs^|dur|jdkrd}|jdkrtd|j|S|j}|S|d|jfvr-td|d|S)z Verifies and adjusts the tensor parallel size for a draft model specified using speculative_draft_tensor_parallel_size. Nrr|zV%s cannot currently be run with tp>1; setting speculative_draft_tensor_parallel_size=1z'speculative_draft_tensor_parallel_size=zB cannot be other value than 1 or target model tensor_parallel_size)r*rr r r)rr4r5rhrhrir.s&    z*SpeculativeConfig._verify_and_get_draft_tpc Cs&t|j||j|j|j|j|jd}|S)zCreate a parallel config for use by the draft worker. This is mostly a copy of the target parallel config, except the tp_size. )rrrmax_parallel_loading_workersdisable_custom_all_reduceray_workers_use_nsightplacement_group)r,rrr6r7r8r9)rr4rrhrhrir0s z.SpeculativeConfig.create_draft_parallel_configr>r:csjdur tdjdkrtdjdjr!jjjdur3jdkr3tdjddg}jd krWjrWtfd d |DsWtd |d jj j S)Nz}num_speculative_tokens must be provided with speculative model unless the draft model config contains an n_predict parameter.rz9Expected num_speculative_tokens to be greater than zero (rsrzmExpect the batch size threshold of disabling speculative decoding is > 1, but got self.disable_by_batch_size=llamaZqwenrc3s|] }|jjjvVqdSrg)rrr*)rZsupported_modelrtrhrir, s z1SpeculativeConfig._verify_args..zEagle3 is only supported for z@ models. Got self.target_model_config.hf_text_config.model_type=) rrrrrrrvrrrr*)ruZeagle3_target_supportedrhrtri _verify_args s6     zSpeculativeConfig._verify_argscCrF)zThe number of additional slots the scheduler should allocate per step, in addition to the slots allocated for each known token. This is equal to the number of speculative tokens, as each speculative token must be scored. )rrtrhrhrinum_lookahead_slots6 sz%SpeculativeConfig.num_lookahead_slotscCs |jdvS)N)rrr)rvrtrhrhri use_eagle@ s zSpeculativeConfig.use_eaglecCs8|j}|dkr dn|jj}|j}d|d|d|dS)Nr zSpeculativeConfig(method=z, model=z, num_spec_tokens=))rvrrr)rurvrZnum_spec_tokensrhrhri__repr__C szSpeculativeConfig.__repr__)1rxryrzrrrr?rrrr{rvSpeculativeMethodrrrrrtr2rrrrrrrrrrr,rrrrrv staticmethodrKr!r8r/r.r0rrr;rr<r=r?rhrhrhrirss  E "    r)rSrrc@seZdZUdZdZeed< dZeed< dZe ed< dZ e eed < d Z e ejefed < d Zeed < eZeeed<dZe eeefed< dZe ed< defddZddZdefddZdefddZdS) LoRAConfigzConfiguration for LoRA. max_lora_rankr| max_lorasFfully_sharded_lorasN max_cpu_lorasrS lora_dtypelora_extra_vocab_sizelora_vocab_padding_sizedefault_mm_loras bias_enabledrncCsvg}||j||j||j||j||j||j||jtj t | dd }|Sr) rrDrErFrHrJrKrMrrr{r\rrrhrhrirvs s       zLoRAConfig.compute_hashcCsd}d}|j|vrtd|jd|d|j|vr&td|jd|d|jdkr4td|jd |jdur?|j|_dS|j|jkrRtd |jd |jd dS) N)rC @rIi@)rIrRzmax_lora_rank (z) must be one of rzlora_extra_vocab_size (r|z max_loras (z) must be >= 1.zmax_cpu_loras (z) must be >= max_loras (r>)rDrrJrErG)ruZpossible_max_ranksZpossible_lora_extra_vocab_sizerhrhrir8 s2         zLoRAConfig.__post_init__ cache_configcCs |jdkr tjstddSdS)Nrz4V0 LoRA does not support CPU offload, please use V1.)cpu_offload_gbrrr)rurSrhrhriverify_with_cache_config s z#LoRAConfig.verify_with_cache_config model_configcCs8|jdvr |j|_dSt|jtrtt|j|_dSdS)N)NrS)rHrrr{rr)rurVrhrhriverify_with_model_config s   z#LoRAConfig.verify_with_model_config) rxryrzrrDr?rrErFrrGrrHrrr LoRADTyperJr4Zget_lora_vocab_padding_sizerKrrLr~r{rMrvr8r#rUrrWrhrhrhrirBM s2      rBc@seZdZUdZeeeefee dZ eeefe d< e edZ eeeeeffe d< dZeeeefe d< dZee d < d Zee d < d Zee d < d efddZded efddZdeeefd eeeffddZdS)rHz+Controls the behavior of multimodal models.rrIrrNrrrFrrrncCrrrrrhrhrirv rzMultiModalConfig.compute_hashmodalitycCs|j|tjr dSdS)zj Get the maximum number of input items allowed per prompt for the given modality. ir|)rIrdrr)rurYrhrhriget_limit_per_prompt s z%MultiModalConfig.get_limit_per_promptinference_kwargscCs|jpi}|t|BS)z Get the keyword arguments to pass to the multi-modal processor according to the extra arguments passed during inference. )rr~)rur[kwargsrhrhrimerge_mm_processor_kwargs s  z*MultiModalConfig.merge_mm_processor_kwargs)rxryrzrrr~r{r?rrrIrrrrrrobjectrrrrrvrZrr]rhrhrhrirH s, "      rHc@seZdZUdZdZeeed< dZee ed< dZ ee ed< dZ ee ed< dZ ee ed< dZee ed< dZeee ed < dZee ed < dZee ed < d efd dZdS)rz:Controls the behavior of output pooling in pooling models.NrP normalize dimensions activationsoftmax step_tag_idreturned_token_idsenable_chunked_processing max_embed_lenrncCrrrrrhrhrirvP rzPoolerConfig.compute_hash)rxryrzrrPrr{rr_rr`r?rarbrcrdr[rerfrvrhrhrhrir s*  r)rrrrrz>Numerical instability. Please use bfloat16 or float32 instead.)Zgemma2gemma3Zplamo2Zglm4r*rcCs|tvr |tjkr dSdS)NFT)_FLOAT16_NOT_SUPPORTED_MODELSrr)r*rrhrhri_is_valid_dtypeu sricCs2|tvr|tjkrt|}td|d|dS)NzThe model type z# does not support float16. Reason: T)rhrrr)r*rreasonrhrhri_check_valid_dtype| s  rkmodel_idr0rcCst|dd}|durt|dd}|dur"t|dr"t|jdd}|dur2t|dr2t|jdd}|durRt||d}|rR|j}rRdd|D}|rRt|S|durYt j }|S)N torch_dtype vision_configrrcSs(h|]}|jD] }|tvrt|qqSrh)Zparameter_count_SAFETENSORS_TO_TORCH_DTYPE)rZfile_mtZ dtype_strrhrhri sz_find_dtype..) rZget_text_configrrnrr?Zfiles_metadatavaluesrHrr)rlr0r config_dtypeZrepo_mtZfiles_mtZ param_dtypesrhrhri _find_dtype s$  rtrsrc sddlm}fdd|jD}|rtj|vrtj}n|d}|tjkr&|}||vr,|S|}|}|dur<|}n |} |d| d}t d||||S)Nrr3csg|] }t|r|qSrh)ri)rrr*rhrirr s z'_resolve_auto_dtype..z (with compute capability r>zHYour device %s doesn't support %s. Falling back to %s for compatibility.) rr4supported_dtypesrrrZget_device_nameget_device_capabilityZas_version_strr r ) r*rsrr4rvZpreferred_dtypeZ device_nameZdevice_capabilityZ device_str version_strrhruri_resolve_auto_dtype s0   ryrocCst|||d}|j}t|tr/|}|dkrt|||d}n |tvr*td|t|}nt|tj r8|}ntd|t ||||krk|tj krVt d|||S|tj krdt d|||St d|||S)NrorS)rzUnknown dtype: zUpcasting %s to %s.zDowncasting %s to %s.zCasting %s to %s.)rtr*rr{r<ry_STR_DTYPE_TO_TORCH_DTYPErrrrkrr r&r )rlr0rrrrsr*rmrhrhrir( s4      r(rrrrr+rrc Cstd}gd}d} |D]} t|| d} | dur%| |kr| n| } t|| }q t|dd} r2d} | }|r@|dur@||kr@d} |}|rM|d|} t|| }|tdkrj|durY|S|dur_|Sd}td|||}t|dd}|durd |jvr|d }|d vr|rtd |d d}|dkr|d}||9}|rd|vr|d}|durt|}t rtd|t|S||krt|dd}|dur||kr|rtdt|Sd|d| d|d|d }t j rtd|t|St |dt|S)z*Get and verify the model's maximum length.inf) Zmax_position_embeddingsZ n_positionsZ max_seq_lenZ seq_lengthmodel_max_lengthZmax_target_positionsZmax_sequence_lengthmax_seq_lengthZseq_lenNr|r+izThe model's config.json does not contain any of the following keys to determine the original maximum length of the model: %s. Assuming the model's maximum length is %d.rrg rope_type)ZsuZlongropeZllama3ztDisabling sliding window is not supported for models with rope_scaling. Please raise an issue so we can investigate.factorg?yarnZ original_max_position_embeddingsr}z--max-model-len is not specified, it's currently using model's default length %s, which might be too large.Please input with --max-model-len based on your request input length and output length, to avoid unnecessary degradation.zDisabling sliding window is not supported for models model_max_length in the config. Please raise an issue so we can investigate.zUser-specified max_model_len (z-) is greater than the derived max_model_len (=z or model_max_length=zR in model's config.json). This may lead to incorrect model outputs or CUDA errors.zD%s Make sure the value is correct and within the model context size.zR To allow overriding this maximum, set the env var VLLM_ALLOW_LONG_MAX_MODEL_LEN=1)rrryrdr r r*rr?r4Zis_tpurZVLLM_ALLOW_LONG_MAX_MODEL_LENr)rrrrr+rrZderived_max_model_lenZ possible_keysZ max_len_keyr*max_lenZ tmp_max_lenZtokenizer_model_max_lengthZdefault_max_lenrr~Zscaling_factorr|r5rhrhrir s         # rrrcCs|s|St|tr |dS|S)a If the input is a non-empty list, the first model_name in `served_model_name` is taken. If the input is a non-empty string, it is used directly. For cases where the input is either an empty string or an empty list, the fallback is to use `self.model`. r)rr[)rrrhrhrir  s   r )rSxgrammarguidanceZoutlinesc@sneZdZUdZdZeed< dZeed< dZ eed< dZ eed< dZ e ed < d e fd d Z d dZdS)DecodingConfigz=Dataclass which contains the decoding strategy of the engine.rSr6Fdisable_fallbackdisable_any_whitespacedisable_additional_propertiesrreasoning_backendrncCrrrrrhrhrirv rzDecodingConfig.compute_hashcCs8|jr |jdvr td|jr|jdkrtddSdS)N)rrzLdisable_any_whitespace is only supported for xgrammar and guidance backends.rzIdisable_additional_properties is only supported for the guidance backend.)rr6rrrtrhrhrir8 s  zDecodingConfig.__post_init__N)rxryrzrr6GuidedDecodingBackendrrrrrrr{rvr8rhrhrhrir s       r)rworkerrzc@seZdZUdZdZeeed< ede fddZ dZ eeed< dZ ee eed< ede fd d Zede fd d Zdefd dZddZddZdS)ObservabilityConfigz6Configuration for observability - metrics and tracing.Nshow_hidden_metrics_for_versionrncCs|jdurdSt|jS)z,Check if the hidden metrics should be shown.NF)rr!Z_prev_minor_version_wasrtrhrhrishow_hidden_metrics s z'ObservabilityConfig.show_hidden_metricsotlp_traces_endpointcollect_detailed_tracescC|jduod|jvpd|jvS)z6Whether to collect model forward time for the request.Nrrzrrtrhrhricollect_model_forward_time   z.ObservabilityConfig.collect_model_forward_timecCr)z6Whether to collect model execute time for the request.Nrrzrrtrhrhricollect_model_execute_time rz.ObservabilityConfig.collect_model_execute_timecCrrrrrhrhrirv rz ObservabilityConfig.compute_hashcCsd|jdurt|jdkrd|jdvr|ddlm}m}|s.|jdur0td|dSdS)Nr|,r)is_otel_availableotel_import_error_tracebackzOpenTelemetry is not available. Unable to configure 'otlp_traces_endpoint'. Ensure OpenTelemetry packages are installed. Original error: )rr_parse_collect_detailed_tracesZ vllm.tracingrrrr)rurrrhrhrir8, s z!ObservabilityConfig.__post_init__cCs0t|jtsJttt|jdd|_dS)Nrr)rrr[rDetailedTraceModulessplitrtrhrhrir9 s  z2ObservabilityConfig._parse_collect_detailed_traces)rxryrzrrrr{rr rrrrr[rrrrvr8rrhrhrhrir s"  r)Z kv_producerkv_both)Z kv_consumerrc@s.eZdZUdZdZeeed< dZeeed< dZ eeed< dZ e ed< dZ ee ed < dZeeed < d Zeed < d Zeed< dZeed< eedZeeefed< dZeeed< defddZd!ddZedefddZedefddZedefddZdefdd ZdS)"KVTransferConfigz0Configuration for distributed KV cache transfer.N kv_connector engine_idrkv_buffer_devicegeAkv_buffer_sizekv_rolekv_rankr|kv_parallel_sizez 127.0.0.1kv_ipi8kv_portrkv_connector_extra_configkv_connector_module_pathrncCrrrrrhrhrirvs rzKVTransferConfig.compute_hashcCsx|jdur tt|_|jdur%|jttvr%td|jdtt|jdur8|jdur:tdttdSdS)NzUnsupported kv_role: z. Supported roles are zLPlease specify kv_disagg_role when kv_connector is set, supported roles are ) rr{uuiduuid4rrKVRolerrrtrhrhrir8 s  zKVTransferConfig.__post_init__cC|jduo |jttvSrg)rrrrrtrhrhriis_kv_transfer_instance   z(KVTransferConfig.is_kv_transfer_instancecCrrg)rrr KVProducerrtrhrhriis_kv_producer rzKVTransferConfig.is_kv_producercCrrg)rrr KVConsumerrtrhrhriis_kv_consumer rzKVTransferConfig.is_kv_consumercCs|j||Srg)rrd)rur*rrhrhriget_from_extra_config rz&KVTransferConfig.get_from_extra_configr) rxryrzrrrr{rrrrrrrrr?rrrrr~rrrrvr8rrrrrrrhrhrhrirE sB       rc@seZdZUdZdZeed< dZeed< dZ eed< dZ e eed < d Z e ed < d Ze ed < d Ze ed< dZeed<dS)KVEventsConfigz&Configuration for KV event publishing.Fenable_kv_cache_eventsnull publisherz tcp://*:5557endpointNreplay_endpointi' buffer_stepsihwmmax_queue_sizertopic)rxryrzrrrrrr{rrrrr?rrrrhrhrhrir s$        rc@seZdZUdZdZeed< eedZ eed< ee dZ e ed< ee dZ e ed< eedZeed< eedZeed < dZeeed < dZeeed < eedZeed < dZeeed < dZeeed< eedZeed< dZee ed< dZ!ee"ed< ee#dZ$e%e#e&fed< dZ'e(ed< de(fddZ)de*de*fddZ+e,ded edeefddZ-e,ded edeefddZ. d1de/d ee0e(ddfd!d"Z1d#d$Z2d%e0de0fd&d'Z3d(d)Z4d*e*fd+d,Z5d-d.Z6d/d0Z7dS)2 VllmConfigzDataclass which contains all vllm-related configuration. This simplifies passing around the distinct configurations in the codebase. NrVrrSrscheduler_configrr~ lora_configrdecoding_configobservability_config quant_configcompilation_configkv_transfer_configkv_events_configadditional_configr instance_idrncCsfg}g}ddlm}|||tj|jr!||jn|d|jr2||jn|d|jrC||jn|d|j rT||j n|d|j re||j n|d|j rv||j n|d|j r||j |t |j jn|d|jr||jn|d|jr||jn|d|jr||jn|d|jr |jr||jn|d|jr||jn|d|jrt|j}tr tjtj|dddd}n|}||n|d||tjt |dddd }|S) rr) __version__NoneT) sort_keysFrN )rrrrrrVrvrSrrrr~rr{max_num_batched_tokensrrrrrrrrr~rrrrr\r)rurZ vllm_factorsrrZadditional_config_hashrrhrhrirv s                     zVllmConfig.compute_hash batch_sizecCs |jj|Srg)rZbs_to_padded_graph_size)rurrhrhripad_for_cudagraphg s zVllmConfig.pad_for_cudagraphcCsddlm}|jdurRddlm}|||}|}|dur9|}||kr9td|jd|d|d| }|j |vrPt|j d |jd ||SdS) zGet the quantization config.rr3N)get_quant_configzThe quantization method z; is not supported for the current GPU. Minimum capability: z. Current capability: rz* is not supported for quantization method z. Supported dtypes: ) rr4rrrrwZto_intZget_min_capabilityrZget_supported_act_dtypesr)rVr~r4rrZcapability_tupleZ capabilityrvrhrhri_get_quantization_confign s4       z#VllmConfig._get_quantization_configcCsddl}t|||Sr)copyrrdeepcopy)rVr~rrhrhriru s z"VllmConfig.get_quantization_configrrcCs6|dur t|}||_t|j}||_t||dS)N)rV)rrrrVrr )rurrrVrhrhriwith_hf_config s    zVllmConfig.with_hf_configcCs||jdur"|j|j|j|j|j|j|j|j|j |j|j dur<|j |j |j |j|j durO|jdurOt|j|j|_ ddlm}|jdurp|jjrp|jjtjkrp|dkrptd|jjdurtjr|jdur|jjstj |j_n tj!|j_ntj!|j_|jj"j#rd|jj"_$|jj"j$r|jj%&d|'r|jj(durtjr|jjtj krt)j |j_(nt)j*|j_(|jdur|jjrt+dt)j*|j_(ntjrd |j_,|-nt)j*|j_(|j j.dkr |jjtj!kr tjs t/d tj!|j_tjs&|j dur&|jjtj!kr&t/d tj!|j_g}|jrV|jj0rV|jj0j1}|dusB|2d krH|&d nt3|jj4ddsV|&d|rv|D]}t+|q[d|j_d|j_5|j durvd|j _6|j7dur|j7j8r|j j6st/d|j7dur|j7j9dkr|j7j8st/d|:|tjr|'r|jj(t)j;kr|jdur|jj<st+dd|j_<|jj(=r|jjtj ksJd|jj(|j>st?dd|_>tjr|jjtj kr|j@tjrZ|jjAs\|Bs|Csd|j_A|jDdur"d|j_A|j7dur,d|j_A|jdur^|jjEdur`|jdurK|jFrKd|j_AdStjGsbt/dd|j_AdSdSdSdSdSdS)z?Verify configs are valid & consistent with each other. Nrr3)r"zTuring devices tensor cores do not support float32 matmul. To workaround this limitation, vLLM will set 'ieee' input precision for chunked prefill triton kernels.Tz +rms_normz&Cudagraph is disabled under eager moder|zWCPU offload is not supported with `torch.compile` in v0 yet. Disabling `torch.compile`.zQLoRA for V0 is not supported with `torch.compile` yet. Disabling `torch.compile`.lastzPOnly "last" pooling supports chunked prefill and prefix caching; disabling both.Z is_causalz_Only models using causal attention supports chunked prefill and prefix caching; disabling both.Fz`KV cache events are on, but prefix caching is not enabled.Use --enable-prefix-caching to enable.rzKV cache events are disabled,but the scheduler is configured to publish them.Modify KVEventsConfig.enable_kv_cache_eventsto True to enable.zaCUDAGraphMode.FULL is not supported with cascade attention currently. Disabling cascadeattention.zxCompilation level should be CompilationLevel.PIECEWISE when cudagraph_mode piecewise cudagraphs is used, cudagraph_mode=r"zThere is a latency regression when using chunked local attention with the hybrid KV cache manager. Disabling it, by default. To enable it, set the environment VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE=1.)Htry_verify_and_update_configrVrrrrrrr~rSrrUrWrrrrr4rchunked_prefill_enabledrrrrwr r)rlevelrrrr( PIECEWISEZNO_COMPILATION pass_configZenable_async_tpenable_sequence_parallelismZ custom_opsrZ is_cuda_alikeZcudagraph_moder)NONEr&Zcudagraph_num_of_warmups_set_cudagraph_sizesrTr rrPr<rrZlong_prefill_token_thresholdenable_prefix_cachingrrrZcheck_and_update_configZFULLrZrequires_piecewise_compilationrrIZset_splitting_ops_for_v1Zdisable_hybrid_kv_cache_managerZis_cudarrrr=Z2VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE)rur4Zdisable_chunked_prefill_reasonsrPrjrhrhrir8 s2                                     zVllmConfig.__post_init__possible_sizescs:fdd|D}|rtd|jjfdd|DS)Ncs g|] }|jjdkr|qSr&rrrsizertrhrirrj zDVllmConfig.update_sizes_for_sequence_parallelism..zkBatch sizes %s are removed because they are not multiple of tp_size %d when sequence parallelism is enabledcs g|] }|jjdkr|qSr&rrrtrhrirrur)r r rr)rurZ removed_sizesrhrtri%update_sizes_for_sequence_parallelismfs  z0VllmConfig.update_sizes_for_sequence_parallelismcs`tjsOg}jdurNjdurNjjsNgdddtddD}jjdkr1jj j r1 |}fdd|D}|rA|dn|d fd d|D}nYg}jdurjjsjj }t |dkrxgdd dtd |ddd D}nt |dkrt|}ntd |djjdkrjj j r |}jjfdd|D}j|dS)a cudagraph batchsize padding logic: `[1, 2, 4] + [8 * i for i in range(1, 1025)]` is a list of all possible batch sizes that cudagraph will capture. Depending on the engine's configuration of `max_num_seqs`, the candidate batch sizes to capture cudagraph will shrink to the subset which just cover the range of `[1, max_num_seqs]`. In the common case, `max_num_seqs` is 256, and the cudagraph batch sizes will be `[1, 2, 4, 8, 16, 24, 32, 40, ..., 256]`. However, if users specify the cudagraph capture sizes through compilation config, we will use the specified sizes instead. In the end, `vllm_config.compilation_config.cudagraph_capture_sizes` will be the final sizes to capture cudagraph (in descending order). During runtime, if batchsize is larger than `vllm_config.compilation_config.cudagraph_capture_sizes`, no cudagraph will be used. If the batch size is no larger than `vllm_config.compilation_config.cudagraph_capture_sizes`, we can quickly find the padded graph size for a given batch size by looking up `vllm_config.compilation_config.bs_to_padded_graph_size`. N)r|rrcSsg|]}d|qS)rNrhr'rhrhrirrrz3VllmConfig._set_cudagraph_sizes..r|icsg|] }|jjkr|qSrh)rZ max_num_seqs)rxrtrhrirrs  rcg|]}|kr|qSrhrhr)max_batchsize_to_capturerhrirr cSsg|]}|qSrhrhr'rhrhrirrsrNz#Invalid value for cuda_graph_sizes=rcrrhrhr)max_num_tokensrhrirrr)rrrrVrr+rrrrrrcuda_graph_sizesrr-rrZinit_with_cudagraph_sizes)ruZbatch_size_capture_listrZ larger_sizesrrh)rrrurirzs^             zVllmConfig._set_cudagraph_sizesrcCs$|j}||}||j_||j_dSrg)rVr,rr)rurrVrhrhrirecalculate_max_model_lens  z$VllmConfig.recalculate_max_model_lencCs|jdurdSt|jddrdSd|j_|jj}|durdSddlm}m}||d}|dur5|||jj r>|||jj dkrQddl m }||dSdS)Nr1FTr)MODELS_CONFIG_MAPHybridAttentionMambaModelConfigrW)SequenceClassificationConfig) rVrr1rmZ!vllm.model_executor.models.configrrrdZverify_and_update_configrrlZ#vllm.model_executor.models.adaptersr)rurmrrrrrhrhrirs$      z'VllmConfig.try_verify_and_update_configcCsdgd|jjd|jd|jjd|jjd|jjd|jjd|jjd |jj d |jj d |jj d |jj d |j jd|j jd|jjd|jjd|jjd|jjd|jjd|jjd|jjd|jd|jd|jjd|jjd|jjd|jj d|jj!d|jj"d|j#S)Nrzmodel=z, speculative_config=z , tokenizer=z, skip_tokenizer_init=z, tokenizer_mode=z , revision=z, override_neuron_config=z, tokenizer_revision=z, trust_remote_code=z, dtype=z, max_seq_len=z, download_dir=z, load_format=z, tensor_parallel_size=z, pipeline_parallel_size=z, disable_custom_all_reduce=z, quantization=z, enforce_eager=z, kv_cache_dtype=z, device_config=z, decoding_config=z, observability_config=z, seed=z, served_model_name=z, enable_prefix_caching=z, chunked_prefill_enabled=z, use_async_output_proc=z, pooler_config=z, compilation_config=)$r!rVrrrrrrrrrrrr~rrrrrr7rrrSZ cache_dtyperrrrrrrrrrrrrtrhrhri__str__st     zVllmConfig.__str__rg)8rxryrzrrVrrrr#rSr,rr-rr rrr~rrrBrrrrrrrrLr'rrrrrr~rrrrrr{rvr?rrArrurKr[rr8rrrrrrhrhrhrir s   [     C T r_current_vllm_config_current_prefixF vllm_configprefixccst}t}ddlm}|j}zdz |a|adVWnty wtd|jj td|jj |rR|jj t j kr]|j|krhtd|jjW|a|atdSW|a|atdSW|a|atdSW|a|atdS|a|atw)a Temporarily set the current vLLM config. Used during model initialization. We save the current vLLM config in a global variable, so that all modules can access it, e.g. custom ops can access the vLLM config to determine how to dispatch. r)compilation_counterNzenabled custom ops: %szdisabled custom ops: %sz`torch.compile` is turned on, but the model %s does not support it. Please open an issue on GitHub if you want it to be supported.)rrZvllm.compilation.counterrnum_models_seen Exceptionr debugrZenabled_custom_opsZdisabled_custom_opsrr(rr rVrget_cached_compilation_config cache_clear)rZ check_compilerZold_vllm_configZ old_prefixrrrhrhriset_current_vllm_configsV            rr|)maxsizecCstjS)zACache config to avoid repeated calls to get_current_vllm_config())get_current_vllm_configrrhrhrhrirEsrcCs(tdurtdddlm}|StS)NzCurrent vLLM config is not set.rr)rr r Z vllm.configrrrhrhrirKs   rcCstdusJdtS)zI Get the prefix of the model that's currently being initialized. Nz!Current model prefix is not set. )rrhrhrhriget_current_model_prefixVs rcCsd}t||}|duS)ak Check if the text looks like a printed Python object, e.g. contains any substring matching the pattern: "at 0xFFFFFFF>" We match against 0x followed by 2-16 hex chars (there's a max of 16 on a 64 bit system). Args: text (str): The text to check Returns: result (bool): `True` if a match is found, `False` otherwise. zat 0x[a-fA-F0-9]{2,16}>N)research)textpatternrVrhrhricontains_object_print_s rcCst|sdStd|)NTzvLLM tried to hash some configs that may have Python objects ids in them. This is a bug, please file an issue. Text being hashed: )rr)rrhrhrirqsrT layer_type layer_namescs4|dur t|jj}|jjfdd|DS)z Get layers from the vLLM config. Args: vllm_config: The vLLM config. layer_type: The type of the layer to get. layer_names: The names of the layers to get. If None, return all layers. Ncs$i|]}t|r||qSrh)r)rZ layer_nameZforward_contextrrhrirs  z/get_layers_from_vllm_config..)r[rZstatic_forward_contextkeys)rrrrhr riget_layers_from_vllm_config}s   r c@s`eZdZUdZdZeed< dZeed< dZ eed< dZ e eed < e d e fd d Zd S)SpeechToTextConfigz(Configuration for speech-to-text models.i> sample_ratemax_audio_clip_sr|overlap_chunk_secondi@min_energy_split_window_sizerncCrrg)rrtrhrhriallow_audio_chunkingrz'SpeechToTextConfig.allow_audio_chunkingN)rxryrzrr rrrr?rrrrrrrhrhrhrir s    r rqc Csi}|D]F\}}t||sJt|d|dt||}t|rHt|sHt|tsCJdt|d|dt|dt|t||}|||<qt|fi|S)Nz has no field ``z Overrides to rz must be a dict or z , but got ) rMrrrrrr~ update_configr )r0rqZprocessed_overrides field_namer current_valuerhrhrirs,   r)NN)FNrg)rrenumrrrrrrcollections.abcr contextlibr dataclassesrrrrrr functoolsr r importlib.utilr typingr rrrrrrrrrrregexrrZpydanticrrrrZpydantic.dataclassesrZsafetensors.torchrrpZtyping_extensionsrrr Z vllm.envsrrr!Zvllm.config.cacher"r#r$r%r&Zvllm.config.compilationr'r(r)r*Zvllm.config.parallelr+r,Zvllm.config.schedulerr-r.Zvllm.config.utilsr/r0Z vllm.loggerr1Z'vllm.model_executor.layers.quantizationr2rr4Zvllm.transformers_utils.configr5r6r7r8r9r:r;r<r=r>r?r@rAZ vllm.transformers_utils.s3_utilsrBZvllm.transformers_utils.utilsrCrDZ vllm.utilsrErFrGrHrIZ _typeshedrJZ transformers.configuration_utilsrKrPZlayersrrtZvllm.model_executor.modelsmodelsrDZ3vllm.model_executor.layers.quantization.base_configrLZ vllm.model_executor.model_loaderrMZ+vllm.model_executor.model_loader.tensorizerrNZvllm.v1.sample.logits_processorrOr~rrZBaseModelLoaderr{globalsrxr rQrr\rrrrr_r[rrarfrrjrqrrr|EnumrrrrrrTrrrrr r r@rrXrBrHrrrrrzrhrrirkrtryr(r?rr rrrrrrrrrrrrrrrrrrrr r rrhrhrhris     4       <         $  4fR<X^`T   ( 4  3  6T ^(E .