o )i@sddlZddlmmZddlmZddlmZddlm Z m Z ddl m Z ddl mZddlmZddlmZd d lmZmZmZmZmZd d lmZeeZejjjj Z!ejj"j#j Z$Gd d d Z%GdddeZ&dS)N)auto_functionalized)PatternMatcherPass)FakeTensorModeunset_fake_temporarily) Attention) VllmConfig) init_logger)current_platform) QUANT_OPS GroupShapeQuantKey empty_bf16 empty_fp32)VllmInductorPassc @sTeZdZ ddedededejfddZdd Zd e d e fd d Z d e fddZ dS)AttentionStaticQuantPatternT layer_name num_heads head_size quant_dtypecCsV||_||_||_||_t|dtj|d|_|jtvs#Jd|jt|j|_ dS)NT)dtypestatic group_shape symmetricz unsupported quantization scheme ) rrrrr r Z PER_TENSOR quant_keyr QUANT_OP)selfrrrrrrh/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/compilation/fusion_attn.py__init__s  z$AttentionStaticQuantPattern.__init__cOs |jdd|}tj|i|S)Ncuda)rZdevice)rtorchempty)rargskwargsrrr empty_quant/sz'AttentionStaticQuantPattern.empty_quantpm_passlayercCs,|j|j|jj|jjr||dSdSN)implZfused_output_quant_supportedrrrr _register)rr&r'rrrregister_if_supported3s z1AttentionStaticQuantPattern.register_if_supportedc sRdtjdtjdtjdtjdtjdtjf fdd }dtjdtjdtjdtjdtjdtjf fd d }tjtNtd jjtd jjtd jjtd jjd jjtd d g}d d}dtj j fdd}t |||||t j |Wdn1swYWddSWddS1swYdS)Nqkv output_attn output_quantscalec s`t|djjg}tt||||jdd}t|ddjjg}tj|||d} | dS)NquerykeyvalueoutputrZ output_scaler )resultinputr1) RESHAPE_OPrrrATTN_OPrr) r,r-r.r/r0r1view_7at1Z attn_out_viewZat2rrrpattern<s( z6AttentionStaticQuantPattern._register..patternc sFt|djjg}tt||||j|d}t|ddjjgS)Nr2r3r )r:rrrr;r)r,r-r.r/r0r1r<r=r>rr replacementRs z:AttentionStaticQuantPattern._register..replacementr csfdd}|S)Ncs|i|Sr(r)r#r$ process_fxtrace_fnrrwrappedqszMAttentionStaticQuantPattern._register..wrap_trace_fn..wrappedr)rCrDrErrBr wrap_trace_fnosz.wrap_trace_fngmcSsddlm}|||S)Nr)view_to_reshape)Z#torch._inductor.fx_passes.post_gradrH)rGrHrrrfx_view_to_reshapevs zAAttentionStaticQuantPattern._register..fx_view_to_reshape)r!ZTensorrrrrrr%rfxZ GraphModulepmZregister_replacementZfwd_only)rr&r?r@ZinputsrFrIrr>rr*:s>   Pz%AttentionStaticQuantPattern._registerN)T) __name__ __module__ __qualname__strintr!rrr%rrr+r*rrrrrs   rcsFeZdZdZdeffdd Zdejjj ddfdd Z d d Z Z S) AttnFusionPassa This pass fuses post-attention quantization onto attention if supported. It uses the pattern matcher and matches each layer manually, as strings cannot be wildcarded. This also lets us check support on attention layers upon registration instead of during pattern matching. Currently, only static fp8 quant is supported, but patterns could easily be added for other quant schemes and dtypes. The bigger hurdle for wider support are attention kernels, which need to support fusing output quant. configcszt||jj|_tdd|_|jD]\}}t||j |j t }| |j|qt|jdkr;tddSdS)NZattn_fusion_pass)Z pass_namerzAttention + quant fusion is enabled, but CompilationConfig.static_forward_context is empty. Cannot access attention layers so no fusion patterns were registered.)superrZcompilation_configZstatic_forward_contextZstatic_fwd_ctxrpatternsitemsrrrr Z fp8_dtyper+lenloggerwarning)rrRr5r'r? __class__rrrs   zAttnFusionPass.__init__graphreturnNcCsD|||d|j|}td|||d|dS)NZbefore_attn_fusionz*Fused quantization onto %s attention nodesZafter_attn_fusion)beginZ dump_graphrTapplyrWdebugZ end_and_log)rr[countrrr__call__s      zAttnFusionPass.__call__cCs t|tSr()rZ hash_sourcerr>rrruuids zAttnFusionPass.uuid) rLrMrN__doc__rrr!rJr[ZGraphrarb __classcell__rrrYrrQs   rQ)'r!Ztorch._inductor.pattern_matcherZ _inductorZpattern_matcherrKZ*torch._higher_order_ops.auto_functionalizerrZtorch._subclasses.fake_tensorrrZvllm.attentionrZ vllm.configrZ vllm.loggerrZvllm.platformsr Zfusionr r r rrZvllm_inductor_passrrLrWopsZvllmZunified_attention_with_outputdefaultr;ZatenZreshaper:rrQrrrrs          g