o )i/@s dZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z ddlmZddlmZmZmZddlZddlZddlmZddlmZdd lmZmZmZdd lmZm Z m!Z!m"Z"dd l#m$Z$dd l%m&Z&m'Z'dd l(m)Z)dZ*e GdddZ+deeddee,dee,de,de,de-de-fddZ.    dHde/ede-de-deeddee,dee,de e0ee-fdffddZ1de/ed e/e"d!e-d"ed#e/e-d$e2e3e-fde0e+e/e,ffd%d&Z4    'dId(e3d)e3d*e3d+e3d,e3d"ede/ed-ee,de-de-d.e5d/e5d0e/e3d#e/e-d1e5d$e2e3e-fd2ee,d3ee e3d4ee2deeddee,dee,d5e,f.d6d7Z6d8d9Z7d:d;Z8de3ddfd?d@Z:dAej;fdBdCZdS)Ja.Benchmark online serving throughput. On the server side, run one of the following commands to launch the vLLM OpenAI API server: vllm serve On the client side, run: vllm bench serve \ --endpoint-type \ --label \ --model \ --dataset-name \ --request-rate \ --num-prompts N)AsyncGeneratorIterable) dataclass)datetime)AnyLiteralOptional)tqdm)PreTrainedTokenizerBase) SampleRequestadd_dataset_parser get_samples)ASYNC_REQUEST_FUNCSOPENAI_COMPATIBLE_BACKENDSRequestFuncInputRequestFuncOutput)wait_for_endpoint)#convert_to_pytorch_benchmark_format write_to_json) get_tokenizerc@seZdZUeed<eed<eed<eed<eed<eed<eed<eed<eed <eed <eeeefed <eed <eed <eed<eeeefed<eed<eed<eed<eeeefed<eed<eed<eed<eeeefed<dS)BenchmarkMetrics completed total_input total_outputrequest_throughputrequest_goodputoutput_throughputtotal_token_throughput mean_ttft_msmedian_ttft_ms std_ttft_mspercentiles_ttft_ms mean_tpot_msmedian_tpot_ms std_tpot_mspercentiles_tpot_ms mean_itl_ms median_itl_ms std_itl_mspercentiles_itl_ms mean_e2el_msmedian_e2el_ms std_e2el_mspercentiles_e2el_msN)__name__ __module__ __qualname__int__annotations__floatlisttupler7r7a/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/benchmarks/serve.pyr1s0 rramp_up_strategy)linear exponentialramp_up_start_rpsramp_up_end_rps request_indextotal_requests request_ratereturnc Csp|r6|dur6|dur6|t|dd}|dkr!|||}||S|dkr/||}|||Std||S)Nr:r;zUnknown ramp-up strategy: )max ValueError) r9r<r=r>r?r@progressZincreaseratior7r7r8_get_current_request_rateOs   rG?input_requests burstinesscs|dks Jd|dt|trt|tst|}t|}|dks'Jdg}g}t|D]/\} } t|||| ||} || | tdkrM|dq/d| |} |tj j || dq/t dt|D]} || || d7<qf|d ur|d dkr||}||d fd d |D}t }t|D])\} } || dkrt }||| |}|dkrt |Id H| || fVqd S) a Asynchronously generates requests at a specified rate with OPTIONAL burstiness and OPTIONAL ramp-up strategy. Args: input_requests: A list of input requests, each represented as a SampleRequest. request_rate: The rate at which requests are generated (requests/s). burstiness (optional): The burstiness factor of the request generation. Only takes effect when request_rate is not inf. Default value is 1, which follows a Poisson process. Otherwise, the request intervals follow a gamma distribution. A lower burstiness value (0 < burstiness < 1) results in more bursty requests, while a higher burstiness value (burstiness > 1) results in a more uniform arrival of requests. ramp_up_strategy (optional): The ramp-up strategy. Can be "linear" or "exponential". If None, uses constant request rate (specified by request_rate). ramp_up_start_rps (optional): The starting request rate for ramp-up. ramp_up_end_rps (optional): The ending request rate for ramp-up. rz4A positive burstiness factor is expected, but given .zNo requests provided.infrH)shapescalerBNcsg|]}|qSr7r7).0delayZnormalize_factorr7r8 szget_request..) isinstancerr5len enumeraterGappendr4nprandomgammarangetimeasynciosleep)rIr@rJr9r<r=r?Z request_ratesZdelay_tsr>requestcurrent_request_ratethetaiZtarget_total_delay_sZstart_tsZ current_tsZsleep_interval_sr7rRr8 get_requestesP !        rcoutputsdur_s tokenizerselected_percentilesgoodput_config_dictcstg}d}d}d} ggg} ggtt|D]g} || jrz|| j} | s2t||| jddj} || ||| j7}d} | dkrY|| j|| j }|| d} | | | || j 7|| j || j|d7}q|dq|rg}g}d|vr|||dt d|vr|| ||dt d|vr|||dt t |D]}t dd t ||D}|r| d7} q|dkrtjd d d td*id |d|dt|d||d| |dt||d|t||dtpdddtpdddtp%dddfdd |Ddtp:dddtpEdddtpPdddfdd |Ddtpeddd tppddd!tp{ddd"fd#d |Dd$tpddd%tpddd&tpddd'fd(d |D}||fS)+aCalculate the metrics for the benchmark. Args: input_requests: The input requests. outputs: The outputs of the requests. dur_s: The duration of the benchmark. tokenizer: The tokenizer to use. selected_percentiles: The percentiles to select. goodput_config_dict: The goodput configuration. Returns: A tuple of the benchmark metrics and the actual output lengths. rF)Zadd_special_tokensrBttfttpote2elcSsg|]\}}||kqSr7r7)rPsrr7r7r8rS sz%calculate_metrics..zYAll requests failed. This is likely due to a misconfiguration on the benchmark arguments.) stacklevelrrrrrrrrrr!r r"c$g|]}|tp d|dfqSrrrXZ percentilerPp)ttftsr7r8rS"r#r%r$r&crprqrrrs)tpotsr7r8rS'rvr'r)r(r*crprqrrrs)itlsr7r8rS,rvr+r-r,r.crprqrrrs)e2elsr7r8rS1rvNr7)r[rUsuccessZ output_tokensgenerated_textZ input_idsrW prompt_lenZlatencyriitl"MILLISECONDS_TO_SECONDS_CONVERSIONzipallwarningswarnrsumrXmeanZstdZmedian)rIrdrerfrgrhactual_output_lensrrZgood_completedZ all_tpotsrb output_lenrjZlatency_minus_ttftZ valid_metricsZ slo_valuesZ req_metricZ is_good_reqmetricsr7)ryrxrwrur8calculate_metricss                       rX endpoint_typeapi_urlbase_urlmodel_id model_namelogprobs disable_tqdmprofileselected_percentile_metrics ignore_eosmax_concurrency lora_modules extra_bodyready_check_timeout_secc9s|tvr t|ntd|tj|pd|pddddddd|vd}tj|dtjd d d }td |dj|dj|dj |dj f\}}}}|duset |t set |t ratd d|DseJdt||||||||||d }t|||dIdH}|jstd|jtdrtfddtt|D| rtdt||||d||||||d }||dIdH} | jrtd| dkrdnd}!|durtd|dtd|d |d!ntd"|td#| d$|!d%td&|| rdntt|d }"|rt|ndfd'd(}#t}$g}%g}&d)}'|dur<|dur<|}'|&|'td*t||| |||2zn3dHW\}(})|durxt |)}*|*|'krxt}+t|'d+|*d+D] },|&|,|+d*qj|*}'|(j|(j|(j |(j f\}-}.}/}0||}1}2rt!}3|3|3}1}2t|1|2|-||.|/||0||d }4|%t"|#|4||"d,qE6tj#|%IdH}5|"dur|"$t|$}6t%||5|6|| |d-\}7td.j&d/d0d1d2td3&d4j'|durtd3&d5||t(d6krtd7&d8|td7&d9|6td3&d:j)td3&d;j*td7&d<j+|r6td7&d=j,td7&d>j-td7&d?j.|6j'j)j*j+|rWj,ndj-j.d@d|5D|7dAd|5DdBd|5DdCd|5DdDd|5DdE|&r|&dF<dGt/dHt/dIt/ffdJdK }8|8dLdMdN|8dOdPdQ|8dRdSdT|8dUdVdWtdX| rtdYt|||dZ|||d[}||dIdH} | jrtd\|$IdHS)]NzUnknown endpoint_type: ri,T<Fzhttps://)limitZlimit_per_hostZ ttl_dns_cacheZ use_dns_cacheZkeepalive_timeoutZenable_cleanup_closedZ force_closessli`T)total) connector trust_envtimeoutz*Starting initial single prompt test run...css|]}t|tVqdSN)rTdict)rPitemr7r7r8 uszbenchmark..z-multi_modal_data must be a dict or list[dict]) modelrpromptrr|rrZmulti_modal_contentrr)Ztimeout_secondsz_Initial test run failed - Please make sure benchmark arguments are correctly specified. Error: z:Initial test run completed. Starting main benchmark run...csg|]}tqSr7)rYchoice)rP_)rr7r8rSszbenchmark..zStarting profiler...z/start_profile)request_func_inputsessionzProfiler startedrHzPoisson processzGamma distributionzTraffic ramp-up strategy: rKzWill increase RPS from z to z( RPS over the duration of the benchmark.zTraffic request rate: zBurstiness factor: z ()zMaximum request concurrency: c sndur|||dIdHS4IdH|||dIdHWdIdHS1IdHs0wYdS)Nrrpbarr7r) request_func semaphorer7r8limited_request_funcs  0z'benchmark..limited_request_funcrO)Zrps timestamprBr)rIrdrerfrgrh {s:{c}^{n}}z Serving Benchmark Result 2=rlncz {:<40} {:<10}zSuccessful requests:zMaximum request concurrency:rL{:<40} {:<10.2f}zRequest rate configured (RPS):zBenchmark duration (s):zTotal input tokens:zTotal generated tokens:zRequest throughput (req/s):zRequest goodput (req/s):z Output token throughput (tok/s):zTotal Token throughput (tok/s):cSg|]}|jqSr7)r|rPoutputr7r7r8rS,cSrr7)rirr7r7r8rS.rcSrr7)r}rr7r7r8rS/rcSrr7)r{rr7r7r8rS0rcSrr7)errorrr7r7r8rS1r)durationrZtotal_input_tokensZtotal_output_tokensrrrr input_lens output_lensrurxgenerated_textserrorsrps_change_eventsmetric_attribute_name metric_name metric_headerc s:|vrdStdj|dddtdd|dtd|d tdd |dtd |d td|d d|d <td |d d |d <td |d d |d <td |d D].\}}t||kr|tt|nt|}tdd|d|d||d|d|d <qldS)Nrr-rrzMean z (ms):Zmean_Z_mszMedian Zmedian_Zstd_Z percentiles_P rtr)printformatgetattrr2str)rrrrtvalueZp_word)rresultrr7r8process_one_metric7s:        z%benchmark..process_one_metricriZTTFTzTime to First TokenrjZTPOTz'Time per Output Token (excl. 1st token)r}ZITLzInter-token LatencyrkZE2ELzEnd-to-end Latencyz2==================================================zStopping profiler...z /stop_profile)rrrr|rrzProfiler stopped)0rrDaiohttpZ TCPConnectorZ ClientSessionZ ClientTimeoutrrr|Zexpected_output_lenZmulti_modal_datarTrr5rrrrzriterr[rUr r] Semaphorer\ perf_counterrWrnow isoformatrcr2next create_taskgathercloserrrr4rrrrrrr)9rrrrrrfrIrr@rJrrrrgrrhrrrr9r<r=rrrZ test_promptZtest_prompt_lenZtest_output_lenZtest_mm_contentZ test_inputZ test_outputZ profile_inputZprofile_output distributionrrZbenchmark_start_timetasksrZ last_int_rpsr_r`Zcurrent_int_rpsrZrps_valrr|rZ mm_contentZ req_model_idZreq_model_nameZreq_lora_modulerrdZbenchmark_durationrrr7)rrrrrrr8 benchmark8s                    '              rc Csxi}gd}|jr:t|j}|D]'\}}||vr*td|d|dt|d|dkr9td|d|dq|S) N)rirjrkzInvalid metric name found, z: z4. The service level objective name should be one of z. rzInvalid value found, z;. The service level objective value should be non-negative.)Zgoodput parse_goodputitemsrDr)argsrhZ VALID_NAMESslo_nameslo_valr7r7r8check_goodput_argsrs" rc CsTi}z|D]}|d\}}t|||<qW|Sty)}ztd|d}~ww)N:zInvalid format found for service level objectives. Specify service level objectives for goodput as "KEY:VALUE" pairs, where the key is a metric name, and the value is a number in milliseconds.)splitr4rDargparseArgumentTypeError)Z slo_pairsrhZslo_pairrrerrr7r7r8rs rrresults file_namecsjgdgdt|fddDfddDd}|r3tj|dd}t||dSdS) N) r rr!Z p99_ttft_msr#r$r%Z p99_tpot_msr(r'r)Z p99_itl_ms)rurxrrcs i|] }|vr||gqSr7r7rPk)rr7r8 s  z4save_to_pytorch_benchmark_format..cs&i|]}|vr|vr||qSr7r7rZignored_metricsrrr7r8rs)rrZ extra_inforz .pytorch.json)rospathsplitextr)rrrZ pt_recordsZpt_filer7rr8 save_to_pytorch_benchmark_formats rparsercCst||jdtdttd|jdtddd|jdtdttd|jd tdd d|jd td d |jdtdd |jdtddd|jdtddd|jdtddd|jdtdd|jddd|jdtdd d|jd!ttd"d#d|jd$td%d&d|jd'dd(d)|jd*dd+d)|jd,dd-d)|jd.dd/d)|jd0dd1d)|jd2dd3d)|jd4d5d6d7d8|jd9tdd:d|jd;tddd)|jd?td@dAd|jdBtdCdDd|jdEdFdGdHdI|dJ}|jdKtddLd|jdMtddNd|jdOtddPd|jdQtddRd|jdStdTgdUdVdW|jdXtddYd|jdZdFdd[d\|jd]tdd^d_gd`dW|jdatddbd|jdctdddd|jdetdfdgddS)hNz--endpoint-typeZopenai)typedefaultchoicesz--labelzkThe label (prefix) of the benchmark results. If not specified, the endpoint type will be used as the label.)rrhelpz --backendZvllmz --base-urlz7Server or API base url if not using http host and port.z--hostz 127.0.0.1)rrz--porti@z --endpointz/v1/completionsz API endpoint.z--max-concurrencyaMaximum number of concurrent requests. This can be used to help simulate an environment where a higher level component is enforcing a maximum number of concurrent requests. While the --request-rate argument controls the rate at which requests are initiated, this argument will control how many are actually allowed to execute at a time. This means that when used in combination, the actual request rate may be lower than specified with --request-rate, if the server is not processing requests fast enough to keep up.z--modelTzName of the model.)rrequiredrz --tokenizerzBName or path of the tokenizer, if not using the default tokenizer.)rrz--use-beam-search store_true)actionz --logprobsaNumber of logprobs-per-token to compute & return as part of the request. If unspecified, then either (1) if beam search is disabled, no logprobs are computed & a single dummy logprob is returned for each token; or (2) if beam search is enabled 1 logprob per token is computedz--request-raterLzNumber of requests per second. If this is inf, then all the requests are sent at time 0. Otherwise, we use Poisson process or gamma distribution to synthesize the request arrival times.z --burstinessrHauBurstiness factor of the request generation. Only take effect when request_rate is not inf. Default value is 1, which follows Poisson process. Otherwise, the request intervals follow a gamma distribution. A lower burstiness value (0 < burstiness < 1) results in more bursty requests. A higher burstiness value (burstiness > 1) results in a more uniform arrival of requests.z--trust-remote-codez"Trust remote code from huggingface)rrz--disable-tqdmz%Specify to disable tqdm progress bar.z --profilezbUse Torch Profiler. The endpoint must be launched with VLLM_TORCH_PROFILER_DIR to enable profiler.z --save-resultz0Specify to save benchmark results to a json filez--save-detailedznWhen saving the results, whether to include per request information such as response, error, ttfs, tpots, etc.z--append-resultz6Append the benchmark result to the existing json file.z --metadataz KEY=VALUE*zKey-value pairs (e.g, --metadata version=0.3.3 tp=1) for metadata of this run to be saved in the result JSON file for record keeping purposes.)metavarnargsrz --result-dirznSpecify directory to save benchmark json results.If not specified, results are saved in the current directory.z--result-filenamezSpecify the filename to save benchmark json results.If not specified, results will be saved in {label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json format.z --ignore-eoszuSet ignore_eos flag when sending the benchmark request.Warning: ignore_eos is not supported in deepspeed_mii and tgi.z--percentile-metricsz ttft,tpot,itlzComma-separated list of selected metrics to report percentils. This argument specifies the metrics to report percentiles. Allowed metric names are "ttft", "tpot", "itl", "e2el". z--metric-percentilesZ99zComma-separated list of percentiles for selected metrics. To report 25-th, 50-th, and 75-th percentiles, use "25,50,75". Default value is "99".Use "--percentile-metrics" to select metrics.z --goodput+FaSpecify service level objectives for goodput as "KEY:VALUE" pairs, where the key is a metric name, and the value is in milliseconds. Multiple "KEY:VALUE" pairs can be provided, separated by spaces. Allowed request level metric names are "ttft", "tpot", "e2el". For more context on the definition of goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 and the blog: https://hao-ai-lab.github.io/blogs/distserve)rrrzsampling parametersz--top-pzHTop-p sampling parameter. Only has effect on openai-compatible backends.z--top-kzHTop-k sampling parameter. Only has effect on openai-compatible backends.z--min-pzHMin-p sampling parameter. Only has effect on openai-compatible backends.z --temperaturezTemperature sampling parameter. Only has effect on openai-compatible backends. If not specified, default to greedy decoding (i.e. temperature==0.0).z--tokenizer-modeauto)rZslowZmistralZcustomzThe tokenizer mode. * "auto" will use the fast tokenizer if available. * "slow" will always use the slow tokenizer. * "mistral" will always use the `mistral_common` tokenizer. *"custom" will use --tokenizer to select the preregistered tokenizer.)rrrrz--served-model-namezoThe model name used in the API. If not specified, the model name will be the same as the ``--model`` argument. z--lora-moduleszA subset of LoRA module names passed in when launching the server. For each request, the script chooses a LoRA module at random.)rrrz--ramp-up-strategyr:r;zThe ramp-up strategy. This would be used to ramp up the request rate from initial RPS to final RPS rate (specified by --ramp-up-start-rps and --ramp-up-end-rps.) over the duration of the benchmark.z--ramp-up-start-rpszcThe starting request rate for ramp-up (RPS). Needs to be specified when --ramp-up-strategy is used.z--ramp-up-end-rpszaThe ending request rate for ramp-up (RPS). Needs to be specified when --ramp-up-strategy is used.z--ready-check-timeout-secrzeMaximum time to wait for the endpoint to become ready in seconds (default: 600 seconds / 10 minutes).) r add_argumentrr5rkeysr2r4add_argument_group)rZsampling_groupr7r7r8 add_cli_argss             rcCstt|Sr)r]run main_async)rr7r7r8mainsrcst|t|jtj|j|jdurV|jtdkr"td|jdus,|j dur0td|jdks:|j dkr>td|j|j krHtd|jdkrV|jdkrVtd|j }|j }|j }|j }|jdurj|jn|j }|j}|jdur|j|j}|j}nd |jd |j|j}d |jd |j}t|||jd } |jdurtd t|| } t|} d d|j|j|j|jdD} | r|jtvrtdd| vrd| d<t !t "t#dFid|j d|d|d|d|d| d| d|j$d|jd|j%d|j&d|j'd|j()d d!d"d#|j*)d Dd$|j+d%| d&|j,d'|j-d(| d)|jd*|jd+|j d,|j.IdH} i}t/01d-}||d.<|j |d<||d/<||d<||d0<|j2|d1<|j3r|j3D]}d2|vr|)d2}|d34||d4<qqtd4|jtdkr|jnd|d<|j%|d<|j,|d&<|jdur|j|d)<|j|d*<|j |d+<i|| }|j5sd5D]}||vr||=|| vr| |=q|j6s|j7r|)d6d7}|j,durd8|j,nd9}|p|}|jdur |d:|jd;|jd<|j d=|d;|d;|d>}n|d;|jd=|d;|d;|d> }|j8r9|j8}|j9rMt:j;|j9d?d@t:j<=|j9|}t>||j7rUdAndBdCdD}|j7rk|?dkrk|@dEtAB||Wdn 1s|wYtC||||S)GNrLzWhen using ramp-up, do not specify --request-rate. The request rate will be controlled by ramp-up parameters. Please remove the --request-rate argument.z_When using --ramp-up-strategy, both --ramp-up-start-rps and --ramp-up-end-rps must be specifiedrz.Ramp-up start and end RPS must be non-negativez+Ramp-up start RPS must be less than end RPSr;z3For exponential ramp-up, the start RPS cannot be 0.zhttp://r)tokenizer_modetrust_remote_codezSPlease specify '--dataset-name' and the corresponding '--dataset-path' if required.cSsi|] \}}|dur||qSrr7)rPrvr7r7r8rs zmain_async..)top_ptop_kmin_p temperaturezESampling parameters are only supported by openai-compatible backends.r grrrrrrfrIrr@rJrrr,rgcSsg|]}t|qSr7)r4rsr7r7r8rSszmain_async..rrhrrrr9r<r=rz %Y%m%d-%H%M%Sdatelabel tokenizer_id num_promptsrrBz5Invalid metadata format. Please use KEY=VALUE format.)rrrurxrr/rOz -concurrencyz -ramp-up-rzqps-Zqpsz.jsonT)exist_okza+wzutf-8)modeencoding r7)DrrYseedrXr9r@r4rDr<r=rrrZserved_model_namerfrrZendpointhostportrrZ dataset_namer rrr r r rbackendrgcZcollectfreezerrrJrrZpercentile_metricsrZmetric_percentilesrrrrrrstrftimermetadatastripZ save_detailedZ save_resultZ append_resultZresult_filenameZ result_dirrmakedirsrjoinopentellwritejsondumpr)rrrrrrrrrrfrIrhZsampling_paramsZbenchmark_resultZ result_jsonZ current_dtrZkvstringfieldZ base_model_idZmax_concurrency_strroutfiler7r7r8rsN                                    6$  r)rHNNN)NNNr)?__doc__rr]rr&rrYr\rcollections.abcrr dataclassesrrtypingrrrrnumpyrXZ tqdm.asyncior Z transformersr Zvllm.benchmarks.datasetsr r r Z)vllm.benchmarks.lib.endpoint_request_funcrrrrZ!vllm.benchmarks.lib.ready_checkerrZvllm.benchmarks.lib.utilsrrZ!vllm.transformers_utils.tokenizerrr~rr2r4rGr5r6rcrrrboolrrr NamespacerArgumentParserrrrr7r7r7r8s          [            <