o
    )ik0                  
   @   sz   d dl mZ d dlZd dlmZ eeZzd dlZW n e	y3 Z
 zede
j W Y dZ
[
ndZ
[
ww G dd dZdS )    )OptionalN)init_loggerzImport error msg: %sc                0   @   s  e Zd Zedejdeejejf fddZedejdejddfddZedejdejddfd	d
Z	edejdejddfddZ
edejdejfddZedejdejfddZedejdejddfddZe					dzdejdejdejdejdededejdejdededeej ded ed!ed"ed#ed$ed%ed&eddf(d'd(Ze					dzdejd)ejd*ejd+ejdejdejdejdededejdejdededeej ded ed!ed"ed#ed$ed%ed&eddf.d,d-Zed.ejdejd/ejd0ed1ejd2eddfd3d4Zed.ejdejd/ejd0ed1ejd2ed5ed6ejddfd7d8Zed9ejd:ejd;edejfd<d=Zed9ejd>ejd:ejd;eddf
d?d@Zedejd/ejdAejdejdBejdCejdeej dDedEedFedGedHedIedJedKejdLedMedNeddf&dOdPZed/ejdAejdejdejdQejded ed!eddfdRdSZe			T	Td{d/ejdAejdejdejdQejded eej d!eej dUedVeddfdWdXZe		Y			Z					d|dejd[ejd\ejd]ejd^ejd_ejdDedEedGed`edaejdeej dbeee  dcee ddeej deedfeej f"dgdhZeejdddddddidjdidddfdkejd^eej dleej dmeej dnee ddfdodpZ edqeej dreej dsejddfdtduZ!edvejdwejdsejddfdxdyZ"dS )}ipex_opsxreturnc                 C   sZ   |  d}|  dd }| |d|} tj| ddd\}}|||}|||}||fS )Nr         )chunksdim)sizeZreshapetorchchunk)r   numdx1Zx2 r   Z/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/_ipex_ops.py_reshape_activation_tensor   s   
z#ipex_ops._reshape_activation_tensoroutNc                 C      t jj||  d S N)ipexllm
functionalsilu_and_mulr   r   r   r   r   r         zipex_ops.silu_and_mulc                 C   r   r   r   r   r   gelu_and_mulr   r   r   r   r   #   r   zipex_ops.gelu_and_mulc                 C   r   r   r   r   r   r   r   gelu_tanh_and_mul'   r   zipex_ops.gelu_tanh_and_mulc                 C      t jj| S r   r   nnr   Zgelur   r   r   r   	gelu_fast+      zipex_ops.gelu_fastc                 C   r    r   r!   r#   r   r   r   gelu_new/   r%   zipex_ops.gelu_newc                 C   r   r   )r   r   r   
gelu_quickr   r   r   r   r'   3   r   zipex_ops.gelu_quickr   @   query	key_cachevalue_cachenum_kv_headsscaleblock_tablescontext_lens
block_sizemax_context_lenalibi_slopeskv_cache_dtypek_scalev_scaletp_rankblocksparse_local_blocksblocksparse_vert_strideblocksparse_block_sizeblocksparse_head_sliding_stepc                 C   sP   |dksJ |  d}|| }tjjj| | |||||||||	|
 d S Nautor   r   r   r   modulesPagedAttentionZsingle_query_kv_attention
contiguousZview_as)r   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   	num_headsnum_queries_per_tokensr   r   r   paged_attention_v17   s    

zipex_ops.paged_attention_v1exp_sum
max_logitstmp_outc                 C   sP   |dksJ |  d}|| }tjjj| | ||||||	|
||| d S r;   r=   )r   rD   rE   rF   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   rA   rB   r   r   r   paged_attention_v2^   s    

zipex_ops.paged_attention_v2	positionskey	head_sizecos_sin_cacheis_neoxc              	   C   s(   | d}tjj| |||||| d S )Nr   )r   r   r   r   rotary_embedding_batched)rH   r)   rI   rJ   rK   rL   rot_dimr   r   r   rotary_embedding   s
   
	zipex_ops.rotary_embeddingrN   cos_sin_cache_offsetsc              
   C   s    t jj| ||||||| d S r   )r   r   r   rM   )rH   r)   rI   rJ   rK   rL   rN   rP   r   r   r   batched_rotary_embedding   s
   z!ipex_ops.batched_rotary_embeddinginputweightepsilonc                 C   s   t jj| ||S r   )r   r   r   rms_norm)rR   rS   rT   r   r   r   rU      s   zipex_ops.rms_normresidualc                 C   s&   t jj|| |d |d}| | d S )NT)r   r   r   Zadd_rms_normZcopy_)rR   rV   rS   rT   tmpr   r   r   fused_add_rms_norm   s   zipex_ops.fused_add_rms_normvalueseqlen_qseqlen_kmax_seqlen_qmax_seqlen_kpdropoutsoftmax_scalezero_tensors	is_causalreturn_softmaxgen_window_size_leftwindow_size_rightlogits_soft_capc                 C   s   t jdr>|dkrtd|d u sJ |dk r|dk sJ t jj|  | | || | |||	|
|||| d S t jj|  | | || | ||||	|
||||||| d S )Ncpu        z)IPEX CPU does not support logits_soft_capr   )	r   __version__endswith
ValueErrorr   r   varlen_attentionr@   int)r)   rI   rY   r   rZ   r[   r2   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   r   r   r   rl      s,   

zipex_ops.varlen_attentionslot_mappingc                 C   (   |dksJ t jjj| |||| d S Nr<   )r   r   r>   r?   reshape_and_cache)rI   rY   r*   r+   rn   r3   r4   r5   r   r   r   rq      s   

zipex_ops.reshape_and_cache      ?k_scale_floatv_scale_floatc
           
      C   ro   rp   )r   r   r>   r?   reshape_and_cache_flash)
rI   rY   r*   r+   rn   r3   r4   r5   rs   rt   r   r   r   ru      s   

z ipex_ops.reshape_and_cache_flashrh   r   qkvcu_seqlens_q	seqused_kcausalblock_tablewindow_sizesoftcapcu_seqlens_k
fa_versions_auxc                 C   s   |d u rt j|dd}t t jdg|jt jd|gt j}|d u r&d}nt|dks.J |d |d f}tj	j
jj| | ||||||||	|
|||d |d dddS )	Nr   )r
   )deviceZdtyper   r   r   rr   )r~   rd   re   r4   r5   )r   ZcumsumcatZtensorr   Zint32tolenr   r   r>   r?   flash_attn_varlen_funcr@   )r   rv   rw   rx   ry   rz   r\   r]   r_   r{   r|   r2   r}   r~   r   Zscheduler_metadatar   Z	q_descaleZ	k_descaleZ	v_descale
num_splitsr   Zreal_window_sizer   r   r   r      s>   
zipex_ops.flash_attn_varlen_funcFr   cache_seqlenscu_seqlens_k_newcache_leftpad	page_sizec                 C   s   t d d S )NzGget_scheduler_metadata is not implemented for ipex_ops, returning None.)loggerZwarning_once)Z
batch_sizer\   r]   Znum_heads_qZnum_heads_kvZheaddimr   Z	qkv_dtypeZ	headdim_vry   r   r   r   Zmax_seqlen_k_newr{   r}   Zhas_softcapr   Zpack_gqaZ	sm_marginr   r   r   get_scheduler_metadata6  s   zipex_ops.get_scheduler_metadata
key_cachesvalue_cachesblock_mappingc                 C      t j| || d S r   )r   xpucopy_blocks)r   r   r   r   r   r   r   R  s
   zipex_ops.copy_blockssrcdstc                 C   r   r   )r   r   swap_blocks)r   r   r   r   r   r   r   \  s   zipex_ops.swap_blocks)r   r   r   r(   r   )NNrr   rr   )
Nrh   NNr   NNNr   N)#__name__
__module____qualname__staticmethodr   ZTensortupler   r   r   r   r$   r&   r'   rm   floatr   strrC   rG   boolrO   rQ   rU   rX   	Generatorrl   rq   ru   listr   Zbfloat16r   r   r   r   r   r   r   r      s   
	
&	
)
	
*		
	

<	
	r   )typingr   r   Zvllm.loggerr   r   r   Zintel_extension_for_pytorchr   ImportErrorewarningmsgr   r   r   r   r   <module>   s   