o )ih@s*dZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z ddl Z ddl Z ddlmZddlmZmZmZddlmZmZmZmZmZmZmZmZmZmZddlmZm Z ddl!m"Z"m#Z#dd l$m%Z%m&Z&dd l'm(Z(dd l)m*Z*dd l+m,Z,dd l-m.Z. d.de/ede0de#de1de2e3e e/e*ff ddZ4 d.de/ede0de#de1de2e3e/e*ff ddZ5  d/de/ede0de"de1de1de3f ddZ6 d.de/ede7dede0de0de1de1de3fdd Z8d!ej9d"e:e7e fddfd#d$Z;d%d&Zfd*d+Z?d!ej9fd,d-Z@dS)0z'Benchmark offline inference throughput.N)AnyOptionalUnion)tqdm)AutoModelForCausalLM AutoTokenizerPreTrainedTokenizerBase) AIMODatasetBurstGPTDatasetConversationDatasetInstructCoderDatasetPrefixRepetitionRandomDataset RandomDataset SampleRequestShareGPTDataset SonnetDatasetVisionArenaDataset)#convert_to_pytorch_benchmark_format write_to_json)AsyncEngineArgs EngineArgs) TextPrompt TokensPrompt) LoRARequest) RequestOutput)BeamSearchParams)merge_async_iteratorsFrequestsn engine_argsdisable_detokenizereturnc s`ddlm}m}|dit|tfdd|Ds!Jdg}g}|D]+}|d|jvr:t|jd|j dnt |j|j d|||d d d |j | d q'd} |j r_d d |D} d} d} | swt } j||| d d} t } n3| dusJddd |D}|dj }|D] }|j |ksJqt } |t||d dt } | | | fS)NrLLMSamplingParamsc3&|]}jjj|j|jkVqdSNZ llm_engine model_config max_model_len prompt_lenexpected_output_len.0requestllmf/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/benchmarks/throughput.py ,  zrun_vllm..pPlease ensure that max_model_len is greater than the sum of prompt_len and expected_output_len for all requests.prompt_token_idsr6multi_modal_datapromptr8?Tr temperaturetop_p ignore_eos max_tokensZ detokenizecSg|]}|jqSr1) lora_requestr,r1r1r2 Gzrun_vllm..F)rBuse_tqdmz$BeamSearch API does not support LoRAcSrAr1)r:r,r1r1r2rCUrD)Z beam_widthr@r?r1)vllmr#r$ dataclassesasdictallappendr:rr8rr+ enable_loratime perf_countergenerateZ beam_searchr)rrrr r#r$promptssampling_paramsr. lora_requestsZuse_beam_searchoutputsstartend output_lenr1r/r2run_vllm$st        rVc sddlm}m}|d it|tfdd|Ds!Jdg}g}|D]}||j|||ddd|j| dq't } j ||dd } t } | | | fS) z Run vLLM chat benchmark. This function is recommended ONLY for benchmarking multimodal models as it properly handles multimodal inputs and chat formatting. For non-multimodal models, use run_vllm() instead. rr"c3r%r&r'r,r/r1r2r3sr4z run_vllm_chat..r5r;Tr<)rENr1) rFr#r$rGrHrIrJr:r+rLrMZchat) rrrr r#r$rOrPr.rSrRrTr1r/r2 run_vllm_chatfs2      rW disable_frontend_multiprocessingc s|ddlm}ddlm}|||d4IdH}|IdHtfdd|Ds.Jdg}g} g} |D]1} |d| jvrIt| jd| j d nt | j| j d | ||d d d | j | d | | j q6g} t } tt|| | D]\}\}}}|j|||d|d}| |qvt| }|2z3dHW\}}q6t }|| WdIdHS1IdHswYdS)Nr)r$)*build_async_engine_client_from_engine_args)rXc3s"|] }j|j|jkVqdSr&)r)r*r+r,r(r1r2r3s z!run_vllm_async..r5r6r7r9r;Tr<test)rBZ request_id)rFr$Z"vllm.entrypoints.openai.api_serverrYZget_model_configrIrJr:rr8rr+rBrLrM enumerateziprNr)rrrrXr r$rYr0rOrPrQr. generatorsrSir:splr generatorZall_gensresrTr1rZr2run_vllm_asyncsp        0rdmodel tokenizermax_batch_sizetrust_remote_codec CsXtj|tj|d}|jjdkr|j|_|}t t |d}t } g} d} d} t t |D]u} || j}|| j}|| j}| |t| |} t| |} t | |krw| t |dkrw|| dj}|| dj}t| |t| |dkrwq.|| ddd j}|j|d|d d d| d }|s|j|dd |t | g} d} d} q.t }|| S) N)Z torch_dtyperhllama)totalriptT)Zreturn_tensorspaddingr;) input_idsZ do_sampleZnum_return_sequencesr=r>Z use_cacheZmax_new_tokens)Zskip_special_tokens)rfrom_pretrainedtorchZfloat16configZ model_typeZ eos_tokenZ pad_tokencudarlenrLrMranger:r*r+rJmaxrnrNZ batch_decodeupdate)rrerfrrgrhr r0ZpbarrSbatchZmax_prompt_lenZmax_output_lenr_r:r*rUZnext_prompt_lenZnext_output_lenrnZ llm_outputsrTr1r1r2run_hfsb         rxargsresultscs\t|dgdgdfdddDd}|r,tj|jdd }t||dSdS) Nrequests_per_secondtokens_per_second)r{r|csi|]}||qSr1r1)r-krzr1r2 sz4save_to_pytorch_benchmark_format..) elapsed_time num_requeststotal_num_tokens)ryZmetricsZ extra_inforz .pytorch.json)rospathsplitext output_jsonr)ryrzZ pt_recordsZpt_filer1r~r2 save_to_pytorch_benchmark_format s  rcCs|j|jd}||j|j|j|j|jd}|jdus|jdkr,|j|d<|j |d<t }n|jdkr=t }|j dkr.r1) rseedrrZ num_promptsrrU dataset_namerandom_range_ratiorrrbackendZ chat_templateZdefault_chat_templaterr rSUPPORTED_DATASET_PATHSr r hf_subsethf_splitr r Zprefix_repetition_prefix_lenZprefix_repetition_suffix_lenZprefix_repetition_num_prefixesZprefix_repetition_output_len ValueErroritemssample)ryrfZ common_kwargsZ sample_kwargsZ dataset_clsr1r1r2 get_requestssp                       rcCs|jdurtjddd|j|_t|dds|j|_hd}|j|vr+td|j|jsF|jsF|j dvrFt d d |_ |j durFtd |j d krct|d ddus[t|dddurctjdddn=|j d kr|jt j tj Bvr|jdksJ|jdn|jtj tj Bvr|jdksJ|jdnt|jd|j d kr|jdurtjddd|j dvr|jdurtjdddt|ddr|jdkrtdt|ddr|jdurtd|jd kr|jdurtd|jd kr|jdurtd|jdvrt|dddurtd |jd!kr|jd"krtd#|jd!kr/|jd$kr/td%|jd!kr@|j|jkrBtd&dSdS)'z* Validate command-line arguments. NzzThe '--dataset' argument will be deprecated in the next release. Please use '--dataset-name' and '--dataset-path' instead.) stacklevelrf>miirrrFzUnsupported backend: >rz?When dataset path is not set, it will default to random datasetrz/input_len must be provided for a random datasetrrrz\--hf-subset and --hf-split will be ignored since --dataset-name is not 'hf'.rz' needs to use vllm-chat as the backend.rFz" needs to use vllm as the backend.z is not supported by hf dataset.zZ--random-range-ratio will be ignored since --dataset-name is not 'random'.>rNrzh--prefix-len will be ignored since --dataset-name is not 'random', 'sonnet', or not set.rKFz4LoRA benchmarking is only supported for vLLM backendz3LoRA path must be provided when enable_lora is Truez,HF max batch size is required for HF backendz)HF max batch size is only for HF backend.>rrZ quantizationz&Quantization is only for vLLM backend.rautoz#dtype must be auto for MII backend.rkzn must be 1 for MII backend.z8Tokenizer must be the same as the model for MII backend.)Zdatasetwarningswarnrgetattrrerfrrrprintrrrkeysr r r rrrhf_max_batch_sizeZdtyper)ryZvalid_backendsr1r1r2 validate_argsYs          rparsercCs|jdtgddd|jdtgdddd |jd tdd d |jd tddd |jdtddd |jdtddd |jdtddd |jdtddd |jdtddd |jdtddd |jdddd d!|jd"ddd#d!|jd$dd%d&|jd'tdd(d |jd)td*d+d |jd,td-d.d |jd/tdd0d |jd1tdd2d |d3}|jd4tdd5d |jd6tdd7d |jd8tdd9d |jd:tdd;d t|}dS)]]]])rrrz--dataset-pathzPath to the datasetz --input-lenz$Input prompt length for each requestz --output-lenzMOutput length for each request. Overrides the output length from the dataset.z--nrkz)Number of generated sequences per prompt.z --num-promptsizNumber of prompts to process.z--hf-max-batch-sizez"Maximum batch size for HF backend.z --output-jsonz3Path to save the throughput results in JSON format.z--async-engine store_trueFz,Use vLLM async engine rather than LLM class.)actionrrz"--disable-frontend-multiprocessingz(Disable decoupled async engine frontend.z--disable-detokenizez[Do not detokenize the response (i.e. do not include detokenization time in the measurement))rrz --lora-pathztPath to the lora adapters to use. This can be an absolute path, a relative path, or a Hugging Face model identifier.z --prefix-lenrzRNumber of fixed prefix tokens before the random context in a request (default: 0).z--random-range-ratiogzRange ratio for sampling input/output length, used only for RandomDataset. Must be in the range [0, 1) to define a symmetric sampling range [length * (1 - range_ratio), length * (1 + range_ratio)].z --hf-subsetzSubset of the HF dataset.z --hf-splitzSplit of the HF dataset.z!prefix repetition dataset optionsz--prefix-repetition-prefix-lenzMNumber of prefix tokens per request, used only for prefix repetition dataset.z--prefix-repetition-suffix-lenz|Number of suffix tokens per request, used only for prefix repetition dataset. Total input length is prefix_len + suffix_len.z --prefix-repetition-num-prefixesz|Number of prefixes to generate, used only for prefix repetition dataset. Prompts per prefix is num_requests // num_prefixes.z--prefix-repetition-output-lenzMNumber of output tokens per request, used only for prefix repetition dataset.) add_argumentstrintfloatadd_argument_groupr add_cli_args)rZprefix_repetition_groupr1r1r2rs  rc Cs|jdur |j|_t||jdurd|_t|jtj|j|jd}t||}t dd|D}d}|j dkr]|j rNt t||jt||j|j}nGt||jt||j\}}n8|j dkry|jdksiJt||j||j|j|j|j}n|j dkrt||jt||j\}}ntd |j |rd}d}|D] }t|tsq||jrt|jnd7}|td d|j D7}q||} ntd d|D} td d|D}| |}|r|j dkrt!d |j dt!dt||dd| |dd||ddt!d|t!d||j"rJ|t|| t||| |d} t#|j"d} t$j%| | ddWdn 1s>wYt&|| dSdS)Nr)rhcss|]}|jduVqdSr&)r8r,r1r1r2r3Iszmain..rFrrkrzUnknown backend: css|] }|rt|jVqdSr&)rsZ token_ids)r-or1r1r2r3ps  css|] }|j|jVqdSr&)r*r+r-rr1r1r2r3tscss|]}|jVqdSr&)r+rr1r1r2r3vsz+WARNING: Multi-modal request with z backend detected. The following metrics are not accurate because image tokens are not counted. See vllm-project/vllm/issues/9778 for details.z Throughput: z.2fz requests/s, z total tokens/s, z output tokens/szTotal num prompt tokens: zTotal num output tokens: )rrrr{r|w)indent)'rfrerrrrrorhranyrZ async_engineuvlooprunrdrrZ from_cli_argsrXr rVrZtensor_parallel_sizerxrrWr isinstancerr6rssumrRrropenjsondumpr) ryrfrZis_multi_modalZrequest_outputsrZtotal_prompt_tokensZtotal_output_tokensrorrzfr1r1r2main>s               r)F)FF)A__doc__argparserGrrrrLrtypingrrrrprrZ transformersrrrZvllm.benchmarks.datasetsr r r r r rrrrrZvllm.benchmarks.lib.utilsrrZvllm.engine.arg_utilsrrZ vllm.inputsrrZvllm.lora.requestrZ vllm.outputsrZvllm.sampling_paramsrZ vllm.utilsrlistrbooltuplerrVrWrdrrx NamespacedictrrrArgumentParserrrr1r1r1r2s 0     F + D ?  >^