o )i~ @spdZddlmZddlmZmZmZmZmZddl Z ddl m Z ddl mZmZmZmZddlmZmZmZmZmZddlmZmZmZmZmZmZdd lm Z m!Z!dd l"m#Z#e#e$Z%Gd d d eZ&eGd ddee!Z'de'de(deefddZ)de'deeede(ddfddZ*Gdddee'Z+Gdddee'Z,de j-de.de j/dee.deef dd Z0dS)!z1Attention layer with xFormers and PagedAttention.) dataclass)DictListOptionalTupleTypeN)ops) AttentionBiasBlockDiagonalCausalMaskBlockDiagonalMask!LowerTriangularMaskWithTensorBias)AttentionBackend AttentionImplAttentionLayerAttentionMetadata AttentionType)CommonAttentionStateCommonMetadataBuilder&get_num_prefill_decode_query_kv_tokensget_seq_len_block_table_argsis_all_cross_attn_metadata_set is_all_encoder_attn_metadata_set)PagedAttentionPagedAttentionMetadata) init_loggerc @seZdZedefddZededfddZededfdd Zeded fd d Z eded fddZ ede de de de de e dff ddZ edejdejdee e fddfddZedeejdejddfdd ZdS)!XFormersBackendreturncCsdS)NZXFORMERSrrrl/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/attention/backends/xformers.pyget_namezXFormersBackend.get_name XFormersImplcCtSN)r!rrrr get_impl_cls"r zXFormersBackend.get_impl_clsrcCr"r#)XFormersMetadatarrrrget_metadata_cls&r z XFormersBackend.get_metadata_clsXFormersMetadataBuildercCr"r#)r'rrrrget_builder_cls*r zXFormersBackend.get_builder_clsrcCr"r#)rrrrr get_state_cls.r zXFormersBackend.get_state_cls num_blocks block_size num_kv_heads head_size.cCst||||Sr#)rget_kv_cache_shape)r*r+r,r-rrrr.2sz"XFormersBackend.get_kv_cache_shape src_kv_cache dst_kv_cache src_to_dstNcCst|||dSr#)r swap_blocks)r/r0r1rrrr2<szXFormersBackend.swap_blocks kv_caches src_to_distscCst||dSr#)r copy_blocks)r3r4rrrr5DszXFormersBackend.copy_blocks)__name__ __module__ __qualname__ staticmethodstrrrr$r&r(r)intrr.torchTensorrr2rr5rrrrrsR     rc@seZdZUdZeejed<eed<eed<e ed<dZ ee eed<dZ eejed<dZ eejed <dZeeed <dZeeed <dZeejed <dZeded <dZeded<dZee eed<dZeejed<dZeejed<dZeeed<dZeeed<dZeejed<dZeejed<ddZeddZeddZededfddZededfdd ZdS)!r%aMetadata for XFormersbackend. NOTE: Any python object stored here is not updated when it is cuda-graph replayed. If you have values that need to be changed dynamically, it should be stored in tensor. The tensor has to be updated from `CUDAGraphRunner.forward` API. seq_lens_tensormax_prefill_seq_lenmax_decode_seq_lenuse_cuda_graphNseq_lens seq_start_loccontext_lens_tensor max_query_lenmax_decode_query_lenquery_start_loc_cached_prefill_metadata_cached_decode_metadataencoder_seq_lensencoder_seq_lens_tensorencoder_seq_start_locmax_encoder_seq_lennum_encoder_tokenscross_slot_mappingcross_block_tablescCsd|_d|_d|_dSr#) attn_biasencoder_attn_biascross_attn_biasselfrrr __post_init__s zXFormersMetadata.__post_init__cCt|S)zO All attention metadata required for encoder attention is set. )rrTrrrrsz1XFormersMetadata.is_all_encoder_attn_metadata_setcCrW)z All attention metadata required for enc/dec cross-attention is set. Superset of encoder attention required metadata. )rrTrrrrsz/XFormersMetadata.is_all_cross_attn_metadata_setrcCs|jdkrdS|jdur|jS|jdus|jdusJ|jdus'|jdus'J|jdur.dn |jd|jd}|jdur?dn |jd|jd}|jdurPdn|jd|j }|jdur_dn|jd|j}|jdurndn|jd|j}|j dur}dn|j d|j}|j durdn|j d|j}t did|jd|j ddd|d|j d|jd |d |d |jd |jd dd|d|d|d|ddd|jd|jd|jd|jd|j|_|jS)Nr num_prefillsnum_prefill_tokensnum_decode_tokens slot_mapping"multi_modal_placeholder_index_mapsenable_kv_scales_calculationrBr>rEr?r@rGrCrD block_tablesrAFrJrKrMrOrPr)rYrHrBrJr>rKrGrCr\rZrDr_r%r]r^rEr?rMrOrP)rUrGrCr\rBr>rDr_rrrprefill_metadatas         z!XFormersMetadata.prefill_metadatacCs6|jdkrdS|jdur|jS|jdus|jdusJ|jdur"dn|j|jd}|jdur1dn|j|jd}|jdur@dn|j|jd}tdiddddd|jd|ddddd |d dd |j d |d |j d|j d|jd|j d|j d|j|_|jjdur|jj}||d|j_|jS)NrrYrZr[r\r]r^Tr>r?r@r_rArJrKrMrOrPr)r[rIr>rKr\rZrYr_r%r@rArJrMrOrPrG)rUr\r>r_qsrrrdecode_metadatash        z XFormersMetadata.decode_metadata) r6r7r8__doc__rr<r=__annotations__r;boolrBrrCrDrErFrGrHrIrJrKrLrMrNrOrPrVpropertyrrr`rbrrrrr%Ls<   9r% attn_metadata attn_typercCsL|tjks |tjkr |jS|tjkr|jS|tjkr|jStdt |)a} Extract appropriate attention bias from attention metadata according to attention type. Arguments: * attn_metadata: Attention metadata structure associated with attention * attn_type: encoder attention, decoder self-attention, encoder/decoder cross-attention Returns: * Appropriate attention bias value given the attention type Invalid attention type rDECODER ENCODER_ONLYrQENCODERrRENCODER_DECODERrSAttributeErrorr:)rgrhrrr_get_attn_bias%s    rprQcCsX|tjks |tjkr||_dS|tjkr||_dS|tjkr#||_dStdt |)ae Update appropriate attention bias field of attention metadata, according to attention type. Arguments: * attn_metadata: Attention metadata structure associated with attention * attn_bias: The desired attention bias value * attn_type: encoder attention, decoder self-attention, encoder/decoder cross-attention riNrj)rgrQrhrrr_set_attn_biasBs       rqc@seZdZeZdS)r'N)r6r7r8r%Z _metadata_clsrrrrr'^sr'c@seZdZdZdejddfdededededee ed eed e d eed e d ee de ddfddZ  dde dejdeejdeejdejdddeejdeejdejfddZejfdejdejdejded e dejf ddZdS) r!a If the input tensors contain prompt tokens, the layout is as follows: |<--------------- num_prefill_tokens ----------------->| |<--prefill_0-->|<--prefill_1-->|...|<--prefill_N-1--->| Otherwise, the layout is as follows: |<----------------- num_decode_tokens ------------------>| |<--decode_0-->|..........|<--decode_M-1-->|<--padding-->| Generation tokens can contain padding when cuda-graph is used. Currently, prompt tokens don't contain any padding. The prompts might have different lengths, while the generation tokens always have length 1. If chunked prefill is enabled, prefill tokens and decode tokens can be batched together in a flattened 1D query. |<----- num_prefill_tokens ---->|<------- num_decode_tokens --------->| |<-prefill_0->|...|<-prefill_N-1->|<--decode_0-->|...|<--decode_M-1-->| Currently, cuda graph is disabled for chunked prefill, meaning there's no padding between prefill and decode tokens. NF num_headsr-scaler, alibi_slopessliding_windowkv_cache_dtypelogits_soft_caprhkv_sharing_target_layer_name use_iroperc Cs| durtd|durtd| rtd||_||_t||_||_|dur2tj |tj d}||_ ||_ ||_ |j|j|_t} || vrUtd|d| d| |_dS)Nz3KV sharing is not supported in V0 XFORMERS backend.zGXFormers does not support logits soft cap. Outputs may be slightly off.zeUsing irope in XFormers is not supported yet, it will fall back to global attention for long context.dtypez Head size z? is not supported by PagedAttention. Supported head sizes are: .)NotImplementedErrorloggerZ warning_oncerrr-floatrsr,r<ZtensorZfloat32rtrurvnum_queries_per_kvrZget_supported_head_sizes ValueErrorrh) rUrrr-rsr,rtrurvrwrhrxryZsupported_head_sizesrrr__init__}s4   zXFormersImpl.__init__layerquerykeyvaluekv_cachergr%output output_scalec Cs|durtd|j} | tjkr|jstd| tjkr#|js#td|d|j |j }|durI|dus6J|d|j |j }|d|j |j }n|dusOJ| tjkr| dkrt ||j |j \} } |dur|dur| tjkrv|j} n|j} t ||| | | |j|j|jt|| \} }}t|}|| d}|d| }|dur|dur|d|}|d|}|jd| ksJ|jd|ksJ|j}r?| dks|j dkr|j||||| d}|j|d| jksJ||d| <nH| tjksJd|jdus J|jdusJt ||||j| | |j|j|j |j|j!|j"|j|j}|d| j|jks9J||d| <|j#}rr| tjksOJd t$|d | \}}}t %|| | ||||j|j |j&|j!|j|j || d<|d|j |j S) ad Forward pass with xFormers and PagedAttention. For decoder-only models: query, key and value must be non-None. For encoder/decoder models: * XFormersImpl.forward() may be invoked for both self- and cross- attention layers. * For self-attention: query, key and value must be non-None. * For cross-attention: * Query must be non-None * During prefill, key and value must be non-None; key and value get cached for use during decode. * During decode, key and value may be None, since: (1) key and value tensors were cached during prefill, and (2) cross-attention key and value tensors do not grow during decode A note on how the attn_type (attention type enum) argument impacts attention forward() behavior: * DECODER: normal decoder-only behavior; use decoder self-attention block table * ENCODER: no KV caching; pass encoder sequence attributes (encoder_seq_lens/encoder_seq_lens_tensor/ max_encoder_seq_len) to kernel, in lieu of decoder sequence attributes (seq_lens/seq_lens_tensor/max_seq_len). Used for encoder branch of encoder-decoder models. * ENCODER_ONLY: no kv_caching, uses the normal attention attributes (seq_lens/seq_lens_tensor/max_seq_len). * ENCODER_DECODER: cross-attention behavior; use cross-attention block table for caching KVs derived from encoder hidden states; since KV sequence lengths will match encoder sequence lengths, pass encoder sequence attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/ max_encoder_seq_len) Args: query: shape = [num_tokens, num_heads * head_size] key: shape = [num_tokens, num_kv_heads * head_size] value: shape = [num_tokens, num_kv_heads * head_size] kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size] NOTE: kv_cache will be an empty tensor with shape [0] for profiling run. attn_metadata: Metadata for attention. attn_type: Select attention type, between encoder attention, decoder self-attention, or encoder/decoder cross- attention. Defaults to decoder self-attention, which is the vLLM default generally Returns: shape = [num_tokens, num_heads * head_size] Nz?fused output quantization is not yet supported for XFormersImplz?Encoder attention requires setting encoder metadata attributes.zUEncoder/decoder cross-attention requires setting cross-attention metadata attributes.r)rhz5Encoder-only models should not have prefix attention.z4Encoder-only models should not have decode metadata.F)'r}rhrrmrrornrviewrrr-r,ZnumelrZsplit_kv_cacherOr\Zwrite_to_paged_cachervZ_k_scaleZ_v_scalerr< empty_likeshaper`r_&_run_memory_efficient_xformers_forwardrlrGrEZforward_prefixr>rtrurbrZforward_decoders)rUrrrrrrgrrrhZ key_cacheZ value_cacheZupdated_slot_mappingZnum_prefill_query_tokensZnum_prefill_kv_tokensZnum_decode_query_tokensZ decode_queryZ prefill_metaoutZ decode_metaZ seq_lens_argZmax_seq_len_argZblock_tables_argrrrforwards>                   zXFormersImpl.forwardc Cs|}|j|jkrP||jd|j|j|jd}|dddddddf|jd|j|j|jd}|dddddddf|jd|j|j|jd}t||}|dur|jdur|tj kr}|j dusjJ|j dusqJt j |j |j |jd}nG|tjkr|j dusJt j |j |jd}n1|tjkr|j dusJt j |j |jd}n|tjkr|j dusJtj |j |jd}ntd||jdur||j}|g}n|tjksJ|j dusJt|j|j|j|j }t||||jdur|d}|d}|d}tj||||dd|jd}||S|j dus"Jt|} d} t |j D]=\} } | | } tj|d| | f|d| | f|d| | f|| d|jd}| | | !||| | | | 7} q.| S)aKAttention for 1D query of multiple prompts. Multiple prompt tokens are flattened in to `query` input. See https://facebookresearch.github.io/xformers/components/ops.html for API spec. Args: output: shape = [num_prefill_tokens, num_heads, head_size] query: shape = [num_prefill_tokens, num_heads, head_size] key: shape = [num_prefill_tokens, num_kv_heads, head_size] value: shape = [num_prefill_tokens, num_kv_heads, head_size] attn_metadata: Metadata for attention. attn_type: Select attention type, between encoder attention, decoder self-attention, or encoder/decoder cross- attention. Defaults to decoder self-attention, which is the vLLM default generally rrN)devicezUnknown AttentionType: %sg)rQprs)"r,rrrrrexpandrprtrrnrBrJr Z from_seqlensrrmrlrkr rruZmake_local_attention_make_alibi_biasr{rqZ unsqueezexopsZ"memory_efficient_attention_forwardrsZview_asr<r enumeratecopy_)rUrrrrgrhZoriginal_queryrQrrstartiseq_lenendrrrrws                    z3XFormersImpl._run_memory_efficient_xformers_forward)NN)r6r7r8rcrrkr;rrrr:rerrr<r=rr%rrrrrr!cs"      4  Ur!rtr,r{rBc Csg}|D]V}tj||d}|dddf|dddf}|ddd}|jd}tjd||||j|dddddddd|f|}||ddddf|t|q|S)NrzrrX)rr{) r<ZarangeremptyrrZmul_appendr ) rtr,r{rBZ attn_biasesrZbiasZ padded_lenrrrrrrs*  r)1rc dataclassesrtypingrrrrrr<ZxformersrrZxformers.ops.fmha.attn_biasr r r r Z vllm.attention.backends.abstractr rrrrZvllm.attention.backends.utilsrrrrrrZvllm.attention.ops.paged_attnrrZ vllm.loggerrr6r~rr%r:rprqr'r!r=r;r{rrrrrs^    0Y   %