o )idS)zg The KV cache is stored inside this class and is accessed via `self.kv_cache`. NautoFr num_heads ($) is not divisible by num_kv_heads ()g?)dtypesinks)r.Zfp8_e5m2z8fp8_e5m2 kv-cache is not supported with fp8 checkpoints.)r-has_sinkzDuplicate layer name: cSsg|]}tgqSr)torchtensor).0_rrr s z&Attention.__init__..)?super__init__sliding_windowZ cache_dtype block_sizeis_attention_freecalculate_kv_scaleskv_cache_dtyper;r<float32_k_scale_v_scale_q_scaleZ _prob_scale_k_scale_float_v_scale_floatr-r$r%r'getr:Zget_quant_method isinstancerr ValueError quant_methodZcreate_weightsget_default_dtyperr1Z get_impl_clsimplrget_namebackendr8rZ is_cuda_alikeZis_cpuuse_direct_callZaccept_output_buffer use_outputr compilation_configZstatic_forward_context layer_namer/rr0rangeZparallel_configZpipeline_parallel_sizekv_cacheenvsZQ_SCALE_CONSTANTq_rangeZK_SCALE_CONSTANTk_rangeZV_SCALE_CONSTANTv_range)selfr$r%r&r'r(r)r*r+r,r-r.r/r0r1Zextra_impl_argsrBrFrCrDrErPr8Zimpl_clsrW __class__rr rAEs             zAttention.__init__querykeyvalue output_shapec Cs|jrtj}|jr|||||jr|dur|n|j}tj||j |j d}|d}|j sZ| d|j |j}| d|j |j}|durM| d|j|j}|durZ| d|j|j}|jrt}|j}t|trm||j}|j|j} |jj||||| ||dn tjj|||||j| d|S|jrt}|j}t|tr||j}|j|j} |j||||| |Stjj||||jS)a_ The KV cache is stored inside this class and is accessed via `self.kv_cache`. Attention metadata (`attn_metadata`) is set using a context manager in the model runner's `execute_model` method. It is accessed via forward context using `vllm.forward_context.get_forward_context().attn_metadata`. N)r8device)output)rEr attn_metadataZenable_kv_scales_calculationcalc_kv_scalesrVshaper;Zzerosr8rfr-viewr$r%r'rUrNdictrXrZvirtual_enginerRforwardopsZvllmunified_attention_with_outputunified_attention) r_rbrcrdrerirhZ hidden_sizeforward_contextZ self_kv_cacherrr rosf         zAttention.forwardcCsv|jt||j|jt||j|jt||j |j |_ |j |_ d|_ dS)NF)rJZcopy_r;absmaxr\rHr]rIr^itemrKrLrE)r_rbrcrdrrr rj"s    zAttention.calc_kv_scalescCs\d|jj}|d|jj7}|d|jj7}|d|jj7}|d|jjj7}|S)Nz head_size=z , num_heads=z, num_kv_heads=z, scale=z , backend=)rRr%r$r'r&ra__name__)r_srrr extra_repr+s zAttention.extra_repr act_dtypecCst|jdr |j||jtjkr@t|jdrBddlm}t|j|s&J|jj durD|jj j t j krF|jj t j |j_ dSdSdSdSdS)Nprocess_weights_after_loadingr9r)FlashInferImpl)hasattrrRr{rTrZFLASHINFER_VLLM_V1Z%vllm.v1.attention.backends.flashinferr|rNr9r8r;rGto)r_rzr|rrr r{3s      z'Attention.process_weights_after_loadingcCs|jSN)r1)r_rrr r@szAttention.get_attn_backendr)rw __module__ __qualname____doc__rZDECODERintfloatrrr rboolstrtyperrAr;TensorSizerorjryr8r{r __classcell__rrr`r r"9sz        I  r"c s\eZdZdZ ddedededeeffdd Zd ej d ej d ej d ej fd dZ Z S)MultiHeadAttentionz7Multi-headed attention without any cache, used for ViT.Nr$r%r&r'cst||_||_||_|dur|n||_|j|jdks,Jd|jd|jd|j|j|_t}t ||dddd}t | }t rOtj|_n|tjtjtjfvr\tj}|tjtjtjhvrh|ntj|_|jtjkr{ts}tj|_dSdSdS)Nrr5r6r7r4F)rFrCrD)r@rAr$r%r&r'num_queries_per_kvr;rQrrrSrZis_rocmr TORCH_SDPAr1Z FLASH_ATTNZFLASH_ATTN_VLLM_V1ZFLEX_ATTENTIONXFORMERSPALLAS_VLLM_V1r!)r_r$r%r&r'r8r1rTr`rr rAGsF         zMultiHeadAttention.__init__rbrcrdr2c CsV|\}}}|d}||||j|j}||||j|j}||||j|j}|j}dkrAtj||dd}tj||dd}|jt j krXddl m } | j ||||jd} nL|jt jkr|dd|||fD\}}}tj||||jd} | dd} n(|jt jkrd d|||fD\}}}dd lm} | ||||jd } | dd} | ||d S) z/Input shape: batch_size x seq_len x hidden_size)dimr)rp)r&cs|] }|ddVqdSrrN transposer=xrrr z-MultiHeadAttention.forward..csrrrrrrr rr)flash_attention)Zsm_scalerg)sizerlr$r%r'rr;Zrepeat_interleaver1rrZxformersrpZ"memory_efficient_attention_forwardr&rFZscaled_dot_product_attentionrrZ$torch_xla.experimental.custom_kernelrZreshape) r_rbrcrdZbszZq_lenr>Zkv_lenZ num_repeatZxopsoutrrrr ropsB        zMultiHeadAttention.forwardr) rwrrrrrrrAr;rrorrrr`r rDs*)rrXcCsJtrtsdSt}t}|j}|durdSt|tsJ||dSr)r r r rrirNrmZwait_for_layer_load)rX connectorrsrirrr wait_for_kv_layer_from_connectors rkv_cache_layercCsRtrtsdSt}t}|j}|durdSt|tsJ|||||dSr)r r r rrirNrmZ save_kv_layer)rXrrrsrirrr maybe_save_kv_layer_to_connectors rrbrcrdr2c Cs`t|t}|j}t|tr||}|j|}|j|j}|j ||||||}t |||Sr rrrirNrmZno_compile_layersrZrnrRror) rbrcrdrXrsrir_rZrhrrr rrs    rrcCst|Sr)r;Z empty_like contiguous)rbrcrdrXrrr unified_attention_fakesr)Zop_nameZop_funcZ mutates_argsZ fake_impl dispatch_keyrh output_scalec Csft|t}|j}t|tr||}|j|}|j|j} |jj ||||| |||dt || dS)N)rhrr) rbrcrdrhrXrrsrir_rZrrr rqs"    rqcCsdSrr)rbrcrdrhrXrrrr "unified_attention_with_output_fakesrr)s           Y     !