o 81 iw@sddlZddlZddlmZddlmZmZddlmZddl m Z m Z m Z m Z ddlZddlmmZddlmZmZddlmZddlmZmZmZz dd lmZmZWneyned d d gZed d d gZYnweGdddZddZ ddZ!d.ddZ"e#         d/ddZ$d.ddZ%e#          d0dd Z&Gd!d"d"Z'ej(fd#e e)e ffd$d%Z*eGd&d'd'Z+e# (   )d1d*d+Z, )d2d,d-Z-dS)3N) namedtuple) dataclassfield)partial)CallableOptionalSequenceUnion) rearrangerepeat)Tensor)ProfilerActivityprofilerecord_function)GreedySearchDecoderOnlyOutputSampleDecoderOnlyOutputr sequencesscoresrc@sdeZdZUdZeed<eed<dZeed<dZeed<ee dZ e ed<d Z e e ed <d d Zd S) InferenceParamszInference parameters that are passed to the main model in order to efficienly calculate and store the context during inference. max_seqlenmax_batch_sizer seqlen_offsetbatch_size_offsetdefault_factorykey_value_memory_dictNlengths_per_samplecCs.||_||_d|_|jdur|jdSdS)Nr)rrrrZzero_)selfrrrg/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/flash_attn/utils/generation.pyreset#s  zInferenceParams.reset)__name__ __module__ __qualname____doc__int__annotations__rrrdictrrrr r rrrrrs    rcCs,|t||ddk}||tddS)z 0 and top_p > 0, then top-k is applied first, then top-p. We assume that all sequences in the same batch have the same length. Arguments: input_ids: (batch, seq_len) max_length: int teacher_outputs (optional): (batch, seq_len). If provided, instead of sampling from the logits, the next token is taken from the teacher_outputs. Useful for testing. Returns: GreedySearchDecoderOnlyOutput or SampleDecoderOnlyOutput, with the following fields: sequences: (batch, max_length) scores: tuples of (batch, vocab_size) Nr5r_decoding_cache)tensor_parallelrrcs|jdk}|rtjdf|jtj|jd}nd}r|s*|||ddjjdd}n j|||jjdd}durC|ddfS|S)Nrr5dtyper< position_idsinference_paramsnum_last_tokensr3.) rr)fulllongr<r-rCrHrun) input_idsrOdecodingrNr-) batch_sizecgmodel vocab_sizerr get_logitss0 zdecode..get_logitscs@dus |jkrt|d}n dd|jf}|dS)Nr.r8rEr5)rrG unsqueeze)r-rOtoken)teacher_output_lenteacher_outputsrEr.r8rr sample_tokenss zdecode..sample_tokenscs<|jdkrdSdur|krdS|jdkrdSdS)NrFTr5)rall) current_tokenrO) eos_token_id max_lengthrr should_stops zdecode..should_stop) enable_timingr(#Prompt processing + decoding time: .0fmsr3rr)rAhasattrrHupdate_graph_cacherOr rr)cudaEvent distributedbarrierrecordappendr synchronizeprintZ elapsed_timerrcattuple)rTrXrdr.r8rErcr_rYrIrWrf seqlen_ogrOrZr`restartendrr output_clsr) rVrWrcrdrXr^r_rEr.r8rYrdecodebsN        r{cCs|j\}}}|d} |j|| |fksJ|j|| fksJ|jtjtjfvs(J|dkr4|dks4Jd|dkr<||n|}|dkrH||n|}|dkrbt||d}t||t||t ||t ||tj |dd} tj |dd} dd } tj || | j d | | || | d d d df|k} | j dd}t|| | jdd}tj| d d d df| dd }tj|| d d dd fgdd}t|jdt|d |ddd}tj|ddjdd}t|d}||d d |f<||dfS)aAlgorithm 1 from [1] [1] Fast Inference from Transformers via Speculative Decoding Yaniv Leviathan, Matan Kalman, Yossi Matias https://arxiv.org/abs/2211.17192 Arguments: logits: Tensor of shape (batch_size, seqlen + 1, vocab_size) logits_draft: Tensor of shape (batch_size, seqlen, vocab_size) tokens_draft: Tensor of shape (batch_size, seqlen) Return: tokens: Tensor of shape (batch_size, seqlen + 1) num_generated_tokens: Tensor of shape (batch_size), with value in [1, seqlen + 1]. For each sequence in the batch, the number of valid tokens that were sampled by speculative sampling. r5r1r2r:rr(r3cSst|jdt|dddS)Nr(z ... -> ... 1r4indexz ... 1 -> ...)r gather)probstokensrrrsz$sample_speculative..r;N)r>z b -> b 1 d)dr|z b 1 d -> b dr=)rr5)rArLr)Zint64int32rDr>r?r0r9r7Zrandr<rawherer%Zargminclamprur r~r rBrCFpad)r-Z logits_draft tokens_draftr.r8rEbatchZ seqlen_p_1rYseqlenrZ probs_draftr~acceptedZ accepted_allZfirst_rejected_idxZ probs_diffZresample_probsZresamplerrrrsample_speculativesF      ""  rc/ s |j\}}|dks Jd|dusJd| r[t|dsd|_t||j|||d| d|_|jj}|||t|ds>d|_t||j|||td|d| d|_|jj}|||n t||d }t||d }d%fd d }d&d d}t|||d}t t fi|}t ||| d}t ||| d}t ||||d}t ||||d}| rddl m }| d}| r| dkrtjtjt}|gg}}d}d} g}!||dkr||dd\}"}#||"||#nt|||d}$|||$d\}%}&| |$7} | r |tj||%gdd|$ddj}'t|&|'ddddf|tj||%gdd||$dd}(|d7}| rQ|tj||%gdd|$ddj})t|(|)t|(|&|%fi|\}"}*|!|*d| rot|"t|*||"ddd|*df||(ddd|*df|*d}+||+d|_|+dkr|jdn|j|_| rtj||dgdd},||,|*dddj}-t|d|-ddddf |j|dkrn.|j|dkr||dddddfdd\}"}#||"||#nt|||jd}$||dddddf|$d\}%}&| |$7} | rM|tj|,|%gdd|$ddj}'t|&|'ddddf|tj|dddddf|%gdd||$dd}(|d7}| r|tj|,|%gdd|$ddj})t|(|)t|(|&|%fi|\}"}*|!|*d| rt|"t|*||"ddd|*df||(ddd|*df|*d}+|j|+7_|+dkr|jdn|j|_| rtj|,|dgdd},||,|*dddj}-t|d|-ddddfq| rL| dkrtjtjtdt|dddtd|td t|!| d!d"d#tj|dd}tj|dd}| rv||j}-t||-dd|ddf|dkr}tnt }.|.||d$S)'a TD: WIP, for my own understanding, lightly tested. Only support batch_size == 1 for now. Speculative decoding, either greedy or with top-k or top-p sampling. If top-k = 0, don't limit the number of candidates (pure sampling). Top-k and top-p can be used together. If top_k > 0 and top_p > 0, then top-k is applied first, then top-p. We assume that all sequences in the same batch have the same length. Arguments: input_ids: (batch, seq_len) max_length: int Returns: GreedySearchDecoderOnlyOutput or SampleDecoderOnlyOutput, with the following fields: sequences: (batch, max_length) scores: tuples of (batch, vocab_size) r5z>Speculative decoding implementation only supports batch_size=1Nz@Speculative decoding implementation doesn't support eos_token_idrH)r5)decoding_seqlensrIrrJFc s|jdk}|r0|jd} tj|jdf|jtj|jd}|dddftj|tj|jd}nd}|r6|s@|||||dj } n||jdksIJ|j |||jdd| df} durg| ddfS| S)Nrr5TrKrM.) rrAr)rQrr<rr@rRr-rHrS) rTrOrXrPrWrUrZ cache_seqlensrNr-rYrrrZPs<      z&decode_speculative..get_logitscSs|dksJ|gg}}t|D]*}|||d|dddf|j|djd7_|||ddqtj|ddddtj|ddfS)aSample `num_tokens` tokens from the model, given the previous logits. Also return the logits of the sampled tokens. Arguments: input_ids: (batch, seqlen) Return: tokens: (batch, num_tokens) scores: (batch, num_tokens), which contains @previous_logits and the logits of the next (num_tokens - 1) tokens. The logits of the last token isn't computed. r5r(Nr3)rangerrrrAr\r)rustack)rT get_logits_fnrO sample_fn num_tokensrrirrrr`us   $z)decode_speculative..sample_tokensr[)rXrW)rrrOr) AutoTokenizerZgpt2)rr3)rPr(TrgirhrizNumber of calls to main model: zAcceptance rate: dz.2f%rj)r5Fr5)!rArkrHrlrOr rrr'rrGZ transformersrZfrom_pretrainedr)rorprmrstimerrr>rur-rtabsmaxritemrsumrr)/rTrXZ model_draftrdZspeculative_lookaheadr.r8rErcrYrIrWrfdebugrVrwZinference_params_draftrOrZr`Zsampling_kwargsrZget_logits_mainZget_logits_draftZsample_tokens_mainZsample_tokens_draftrZ tokenizerrxrrZnum_main_model_callsZnum_draft_tokensZnum_accepted_tokens_historyrZ scores_newZ n_spec_tokensrZ scores_draftZscores_draft_refr-Z logits_refZnum_generated_tokensZ num_generatedZcur_idsZ scores_refrzrrrdecode_speculative sn !        %        $    ($    $$    (D   " ( rc@s*eZdZd ddZ     d dd ZdS) GenerationMixinNcKstN)NotImplementedError)rrVrrLkwargsrrrallocate_inference_cache7sz(GenerationMixin.allocate_inference_cacher5r1r2Fc Ks4t|||f|||d|} |sd| _|r| S| jS)Nr[)r{rr) rrTrdr.r8rEZreturn_dict_in_generateZ output_scoresroutputrrrgenerate:s zGenerationMixin.generater)r5r1r2FF)r!r"r#rrrrrrr6s rlayerscsNtjtjtjfvs J||d||ft|trt|}fdd|DS)Nrcsi|] }|tjdqS))r<rL)r)empty).0rr<rLZkv_cache_shaperr Zsz,allocate_inference_cache..)r)float16Zbfloat16Zfloat32 isinstancer%r)rrZnheadsheaddimrr<rLrrrrMs   rc@sdeZdZUdZeed<dZeed<dZdZe e dZ e ed<dZ dZ eeed<dZeeed<dS) DecodingCGCacherrrNr callablesrOrS)r!r"r#rr%r&rr<rLrr'rmempoolrOrrrSrrrrrr]s  rrrc  svdurttt|} | j} |dur| j}| |fjjfks-|jks-|jkri_d_ d_ t | |__||__t |drU||||} nt|jd|jj|jj} t|||jj|| |jj| |} tj|f|tj| d} t|||| | d_ tjj_ |D]}||fjvrt|j |||j |dj||f<qfdd}|_dj _S) NrZhead_dimrK)rrrrr)decoding_seqlenr n_warmupscs(|jdd\}}j||f|||S)Nr)rAr)rTrNrrVrcacherrdispatchsz$update_graph_cache..dispatchr)rnextiter parametersr<rLrrrrrOgcZcollectrkrgetattrconfigZ hidden_sizeZnum_attention_headsZnum_hidden_layersr)rQrrrmZgraphsZgraph_pool_handle capture_graphrSr)rXrrVrwrrrIrLrZ param_exampler<Z inf_cacherrrrrrrrlisp       rlc s^tt|j}tj||fdtj|dtj||fdtj|dj}||_jjdd<tj } | tj tj | &t|D] } ||djqI| tjrctjWdn1smwYtj | tj tj j|d||djWdn1swYfdd} |_| S)NrrKrM)poolcs2|jdd<||Sr)rZcopy_ZreplayrD)Z new_input_idsZnew_position_idsrgraphrOrTr-rNrrrSs   zcapture_graph..run)rrrr<r)rQrRrrrmZStreamZ wait_streamZcurrent_streamstreamrr-rsroZis_initializedrpZ CUDAGraphr) rXrOrVrrrrr<Zseqlen_offset_ogs_rSrrrrsL      r)r5r1r2) r5r1r2NNNr5FF) rr5r1r2NNr5FFF)rr5Nr)r5Nr).rr collectionsr dataclassesrr functoolsrtypingrrrr r)Ztorch.nn.functionalnnZ functionalrZeinopsr r r Ztorch.profilerr rrZtransformers.generationrr ImportErrorrr0r9rGZinference_moder{rrrrr%rrrlrrrrrs~      n; +    L