o )i @sUddlZddlZddlZddlZddlZddlZddlZddlmZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZddlmZmZddlmZddlmZddlmZddlmZmZmZmZddlZddlZddl Z!ddl"Z"ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)dd l*m+Z+dd l,m-Z-dd l.m/Z/m0Z0m1Z1dd lm2Z2dd l3m4Z4ddl5m6Z6ddl7m8Z8m9Z9m:Z:m;Z;ddlm?Z?m@Z@mAZAmBZBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZKddlLmMZMddlNmOZOddlPmQZQddlRmSZSddlTmUZUmVZVmWZWddlXmYZYddlZm[Z[ddl\m]Z]m^Z^ddl_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{ddl|m}Z}ddl~mZdd lmZdd!lmZdd"lmZdd#lmZmZmZdd$lmZdd%lmZdd&lmZdd'lmZdd(lmZmZdd)lmZdd*lmZmZmZdd+lmZmZmZmZdd,lmZdd-lmZdd.lmZdd/lmZdd0lmZdd1lmZmZmZmZmZmZdd2lmZdd3lmZe jed4<ed5ZeZeejed6<ed7e&fd8d9Zeejddd:d;edeeeefd?eeSf d@dAZeejdBdd:dCeKdeeeefd?eeSf dDdEZdFe)fdGdHZe$ZGdIdJdJe0Zd7e&fdKdLZdMe)d?efdNdOZdMe)d?efdPdQZdMe)d?eefdRdSZdMe)d?ee}fdTdUZdMe)d?eefdVdWZdMe)d?eefdXdYZdMe)d?eefdZd[ZdMe)d?eefd\d]ZdMe)d?eefd^d_ZdMe)d?eefd`daZdMe)d?efdbdcZdMe)d?efdddeZdMe)d?efdfdgZdMe)d?eSfdhdiZejdje0dkdFe)d?e0fdldmZeĠdndMe)fdodpZejdqe0dkejdqe0dkdFe)d?e0fdrdsZejdte%eÃgejjduekiejjduekiejjduekiejjduekiidvedMeudFe)fdwdxZejdye%eÃgejjduekiejjduekiejjduekiidvedMefdFe)fdzd{Zd|d}ZeĠd~dFe)fddZeĠdddZejde%eÃgejjddiiiejjduekiejjduekiejjduekiidvedMeqdFe)fddZeĠddedFe)fddZeĠddedFe)fddZejde%eÃgejjddiiiejjduekiejjduekiejjduekiidveedMe`dFe)fddZejde%eÃgejjddiiiejjduekiejjduekiejjduekiidveedMeddFe)fddZejde%eÃgejjduekiejjduekiidveedMehdFe)fddZejde%eÃgejjduekiejjduekiidveedMemdFe)fddZejde%eÃgdeedMebdFe)fddZejde%eÃgejjduekiejjduekiidveedMesdFe)fddZejde%eÃgejjduekiejjduekiidveedMesdFe)fddZejdejjddiiiejjduekiejjduekiejjduekiideedFe)dMeewe'ffddZejdejjddiiiejjduekiejjduekiejjduekiideedMeeye'fdFe)fddZejde%eÃgejjduekiejjduekiidveedMeodFe)fddZejde%eÃgejjduekiejjduekiidvedMeodFe)fddZejde%eÃgejjduekiejjduekiidvedMeodFe)fddZeGjredeĠddFe)fddZeĠddFe)fddZeĠddFe)fddZeĠddFe)fdd„ZeĠdádFe)fddńZejde%eÃgejjdueiejjduekiejjduekiejjduekiidvdFe)fddȄZeĠdɡdFe)fdd˄ZeZee)geefZeee)geefZe`eeffedeeffeheeffebeeffeseeffeoeeffemeeffgZeeeeeeffed<dd΄eDZejde%eÃgejjduekiejjduekiejjduekiidvdFe)fddфZeGjredҡeĠdӡdFe)fddՄZ eĠd֡dFe)fdd؄Z eGj r ed١ejde%eÃgddMeldFe)fdd܄Z ejde%eÃgddMe{dFe)fdd߄Z deed?eefddZGdddZGdddZdBaGdddZded?efddZGdddZded?dfddZded?dfddZd;ed?e&fddZdieSdeIde;d;ed?df ddZdeeefd?e j fddZded?e j fddZddZddZd ddZ d d ddZe dkreedd Z!e]e!Z!e!"Z#e^e#e"$ee#dSdS( N) Namespace) AsyncIterator Awaitable)asynccontextmanager)partial) HTTPStatus) AnnotatedAnyCallableOptional) APIRouterDependsFastAPIForm HTTPExceptionRequest)RequestValidationError)CORSMiddleware) JSONResponseResponseStreamingResponse) make_asgi_app)Instrumentatoriterate_in_threadpool)URLHeadersMutableHeadersState)Mount)ASGIAppMessageReceiveScopeSend) assert_never) VllmConfig)AsyncEngineArgs)AsyncLLMEngine)MQLLMEngineClient) run_mp_engine) EngineClient)load_chat_templateresolve_hf_chat_templateresolve_mistral_chat_template) serve_http) RequestLogger)make_arg_parservalidate_parsed_serve_args)ChatCompletionRequestChatCompletionResponseClassificationRequestClassificationResponseCompletionRequestCompletionResponseDetokenizeRequestDetokenizeResponseEmbeddingRequestEmbeddingResponse ErrorInfo ErrorResponseLoadLoRAAdapterRequestPoolingRequestPoolingResponse RerankRequestRerankResponseResponsesRequestResponsesResponse ScoreRequest ScoreResponseTokenizeRequestTokenizeResponseTranscriptionRequestTranscriptionResponseTranslationRequestTranslationResponseUnloadLoRAAdapterRequest)OpenAIServingChat)ServingClassification)OpenAIServingCompletion)OpenAIServingEmbedding) OpenAIServing) BaseModelPathLoRAModulePathOpenAIServingModels)OpenAIServingPooling)OpenAIServingResponses) ServingScores)OpenAIServingTokenization)OpenAIServingTranscriptionOpenAIServingTranslation)ToolParserManager)DemoToolServer MCPToolServer ToolServer) cli_env_setupload_aware_calllog_non_default_argswith_cancellation) init_logger)ReasoningParserManager)(maybe_register_config_serialize_by_value)MistralTokenizer) UsageContext)DeviceFlexibleArgumentParser decorate_logsget_open_zmq_ipc_pathis_valid_ipv6_address set_ulimit)get_prometheus_registry) __version__prometheus_multiproc_dirz"vllm.entrypoints.openai.api_server_running_tasksappcszG|jjr"|jjfdd}t|}t||tjnd}t t z dVW|dur9| n |durC| wwW|`dS|`w)Ncs& ttjIdHIdHqN)asynciosleepenvsZVLLM_LOG_STATS_INTERVALZ do_log_stats engine_clientryn/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py _force_logs zlifespan.._force_log) state log_statsr{rv create_taskrsaddadd_done_callbackremovegcZcollectfreezecancel)rtr}taskryrzr|lifespanys*    r usage_context disable_frontend_multiprocessing client_configargsrrrreturnc Cstddkr!tdtdtdgttdt |}|dur/t |j }t ||||d4IdH}|VWdIdHdS1IdHsPwYdS)NZVLLM_WORKER_MULTIPROC_METHOD forkserverz!Setup forkserver with pre-importszvllm.v1.engine.async_llmzForkserver setup complete!r)osgetenvloggerdebugmultiprocessingZset_start_methodZset_forkserver_preloadrZensure_runningr'Z from_cli_argsboolr*build_async_engine_client_from_engine_args)rrrr engine_argsZengineryryr|build_async_engine_clients*     .rFrc s|j|d}tjrW|rtdddlm}d}|r |dnd}|r)|dnd}z#|j|||j |j |||d }| IdH|VW|rM| dSdS|rV| wwt |s^|rd} z tj|||j |j d } | VW| r}t| d r| dSdSdS| rt| d r| wwwd tjvrtatjtjd <ntd ttdtd} ttjdddd} | jt|tj |j |j | fd} | !| j"} | dusJdt#d| fdd}t$%|t&t || }t'()d|IdH}zN z |*IdHWnt+y"| ,r| j-s t.ddYnwq|VW| /|0| 1d| j2dur@| 3ddl4m5}|6| j"dS| /|0| 1d| j2dure| 3ddl4m5}|6| j"w)z Create EngineClient, either: - in-process using the AsyncLLMEngine Directly - multiprocess using AsyncLLMEngine RPC Returns the Client or None if the creation failed. )rzrV1 is enabled, but got --disable-frontend-multiprocessing. To disable frontend multiprocessing, set VLLM_USE_V1=0.r)AsyncLLMN client_count client_index) vllm_configrenable_log_requestsdisable_log_statsZclient_addressesrr)rrrrshutdownZPROMETHEUS_MULTIPROC_DIRzFound PROMETHEUS_MULTIPROC_DIR was set by user. This directory must be wiped between vLLM runs or you will find inaccurate metrics. Unset the variable and vLLM will properly handle cleanup.z0Multiprocessing frontend to use %s for IPC Path.ZspawnbTF)lock)targetrzEngine process failed to start.z"Started engine process with PID %dcs*dd}tj|rt|dSdS)Nzipc://)replacerpathexistsr)Z socket_pathZipc_pathryr|_cleanup_ipc_path)s  zEbuild_async_engine_client_from_engine_args.._cleanup_ipc_pathzCEngine process failed to start. See stack trace for the root cause.) multiprocess)7Zcreate_engine_configrx VLLM_USE_V1rwarningZvllm.v1.engine.async_llmrpopZfrom_vllm_configrrZreset_mm_cacherr)Zis_unsupported_configr(hasattrrenvirontempfileTemporaryDirectoryrrnamermrrZ get_contextrgValueProcessr*riOPENAI_API_SERVERstartpidinfoatexitregisterrrvget_running_looprun_in_executorsetup TimeoutErroris_alivevalue RuntimeError terminateclosejoinexitcodekillprometheus_clientrZmark_process_dead)rrrrrrZ async_llmrrr{contextZ engine_aliveZengine_processZ engine_pidrZ build_clientZmq_engine_clientrryrr|rs                      r raw_requestcs>|jdd}|jdddd}|dkrtdgd dS) N content-typer;r)maxsplitrzapplication/jsonz:Unsupported Media Type: Only 'application/json' is allowederrors)headersgetlowersplitr)r content_type media_typeryryr|validate_json_requestXsrc@seZdZejZdS)PrometheusResponseN)__name__ __module__ __qualname__rZCONTENT_TYPE_LATESTrryryryr|rds rcCsVt}tgd|d|j|tdtdt|d}t d|_ |j |dS)z*Mount prometheus metrics to a FastAPI app.)/metrics/health/load/ping/version /server_info)Zexcluded_handlersregistryresponse_classr)rz^/metrics(?P.*)$N) rprrZ instrumentZexposerrrrecompileZ path_regexZroutesappend)rtrZ metrics_routeryryr| mount_metricshs  rrequestcCst|Sru) tokenizationrryryr|basercC |jjjSru)rtr~openai_serving_modelsrryryr|models rcCrru)rtr~openai_serving_responsesrryryr| responsesrrcCrru)rtr~openai_serving_chatrryryr|chatrrcCrru)rtr~openai_serving_completionrryryr| completionrrcCrru)rtr~openai_serving_poolingrryryr|poolingrrcCrru)rtr~openai_serving_embeddingrryryr| embeddingrrcCrrurtr~openai_serving_scoresrryryr|scorerrcCrru)rtr~openai_serving_classificationrryryr|classifyrrcCrrurrryryr|rerankrrcCrru)rtr~openai_serving_tokenizationrryryr|rrrcCrru)rtr~openai_serving_transcriptionrryryr| transcriptionrrcCrru)rtr~openai_serving_translationrryryr| translationrrcCrru)rtr~r{rryryr|r{rr{rrcst|IdHtddS)z Health check.N status_code)r{Z check_healthrrryryr|healths r rcstd|jjjidS)NZ server_loadcontent)rrtr~server_load_metricsrryryr|get_server_load_metricss rrcst|IdHS)z+Ping check. Endpoint required for SageMakerN)r r ryryr|pingsrz /tokenizemodel) dependenciesrc st|}z |||IdH}Wn-ty'}z ttjjt|d|d}~wty=}z ttj jt|d|d}~wwt |t rMt | |jjdSt |trYt | dSt|dS)Nrdetailr rr )rZcreate_tokenizeNotImplementedErrorrrNOT_IMPLEMENTEDrstr ExceptionINTERNAL_SERVER_ERROR isinstancer>r model_dumperrorcoderIr%rrhandler generatoreryryr|tokenizes4   r"z /detokenizec st|}z |||IdH}Wn+ty%}z tt|gd|d}~wty;}z ttjj t|d|d}~wwt |t rKt | |jjdSt |trWt | dSt|dS)Nrrrr )rZcreate_detokenize OverflowErrorrrrrrrrrr>rrrrr:r%rryryr| detokenizes,   r$cCs,t|ddrtddtfdd}dSdS)z>Conditionally register the tokenizer info endpoint if enabled.Zenable_tokenizer_info_endpointFz/tokenizer_inforcs:t|IdH}t|t|tr|jjdSddS)z(Get comprehensive tokenizer information.Nrr)rget_tokenizer_inforrrr>rr)rresultryryr|r%,s zBmaybe_register_tokenizer_info_endpoint..get_tokenizer_infoN)getattrrouterrr)rr%ryryr|&maybe_register_tokenizer_info_endpoint(s r)z /v1/modelscs&t|}|IdH}t|dS)Nr )rshow_available_modelsrr)rrZmodels_ryryr|r*5sr*rcsdti}t|dS)Nversionr ) VLLM_VERSIONr)verryryr| show_version=s r.z /v1/responsesr text/event-streamcvt|}|durt|jddS|||IdH}t|tr)t||jj dSt|t r5t|dSt |ddS)N(The model does not support Responses APImessagerr r/r r) rrcreate_error_responsecreate_responsesrr>rrrrrErrrrr ryryr|r6Cs   r6z/v1/responses/{response_id} response_idc^t|}|durt|jddS||IdH}t|tr(t||jj dSt|dSNr1r2rr ) rrr5retrieve_responsesrr>rrrrr8rrresponseryryr|r;f r;z"/v1/responses/{response_id}/cancelcr9r:) rrr5cancel_responsesrr>rrrrr<ryryr|r?ur>r?z/v1/chat/completionscr0)Nz/The model does not support Chat Completions APIr2rr r/r4) rrr5create_chat_completionrr>rrrrr4rr7ryryr|r@s   r@z/v1/completionsc st|}|durt|jddSz |||IdH}Wn-ty3}z ttjjt |d|d}~wt yI}z ttj jt |d|d}~wwt |t rYt||jjdSt |tret|dSt|ddS)Nz*The model does not support Completions APIr2rrr r/r4)rrr5create_completionr#rr BAD_REQUESTrrrrrr>rrrrr8rrryryr|rAs<   rAz/v1/embeddingscvt|}|durt|jddS|||IdH}t|tr)t||jj dSt|t r5t|dSt |dS)Nz)The model does not support Embeddings APIr2rr ) rrr5create_embeddingrr>rrrrr<r%r7ryryr|rDs    rDz/poolingcrC)Nz&The model does not support Pooling APIr2rr ) rrr5create_poolingrr>rrrrrAr%r7ryryr|rE    rEz /classify)rcrC)Nz-The model does not support Classification APIr2rr ) rrr5create_classifyrr>rrrrr6r%r7ryryr|rGs   rGz/scorecrC)Nz$The model does not support Score APIr2rr ) rrr5 create_scorerr>rrrrrGr%r7ryryr|rH%rFrHz /v1/scorectdt||IdHS)NzTo indicate that Score API is not part of standard OpenAI API, we have moved it to `/score`. Please update your client accordingly.)rrrHrrryryr|create_score_v1As  rKz/v1/audio/transcriptions)rcst|}|durt|jddS|jIdH}||||IdH}t|tr2t| |j j dSt|t r>t| dSt |ddS)Nz-The model does not support Transcriptions APIr2rr r/r4)rrr5filereadZcreate_transcriptionrr>rrrrrKr)rrr audio_datar ryryr|create_transcriptionsU"   rOz/v1/audio/translationscst|}|durt|jddS|jIdH}||||IdH}t|tr2t| |j j dSt|t r>t| dSt |ddS)Nz+The model does not support Translations APIr2rr r/r4)rrr5rLrMZcreate_translationrr>rrrrrMr)rrrrNr ryryr|create_translations~rPrQz/rerankcrC)Nz-The model does not support Rerank (Score) APIr2rr ) rrr5 do_rerankrr>rrrrrCr%r7ryryr|rRs    rRz /v1/rerankcrI)NzTo indicate that the rerank API is not part of the standard OpenAI API, we have located it at `/rerank`. Please update your client accordingly. (Note: Conforms to JinaAI rerank API))rZ warning_oncerRrJryryr| do_rerank_v1s  rSz /v2/rerankcst||IdHSru)rRrJryryr| do_rerank_v2s rTz[SECURITY WARNING: Development endpoints are enabled! This should NOT be used in production!rcsdt|jjji}t|dS)Nrr )rrtr~rr)rZ server_inforyryr|show_server_infos rUz/reset_prefix_cachecsTd}|jd}|durt|}tdt|t||IdHt ddS)z Reset the prefix cache. Note that we currently do not check if the prefix cache is successfully reset in the API server. Ndevicez*Resetting prefix cache with specific %s...rr) query_paramsrrjupperrrrr{reset_prefix_cacher)rrVZ device_strryryr|rYs   rYz/sleepcs2|jdd}t|t|IdHtddS)Nlevel1rr)rWrr{rwintr)rrZryryr|rws rwz/wake_upcsD|jd}|gkr d}td|t||IdHtddS)Ntagsz wake up the engine with tags: %srr)rWgetlistrrr{wake_upr)rr]ryryr|r_s   r_z /is_sleepingcs,tdt|IdH}td|idS)Nz$check whether the engine is sleeping is_sleepingr )rrr{r`r)rr`ryryr|r`s r`z/scale_elastic_epc sFz |IdH}Wntjy}ztddd|d}~ww|d}|dd}|dur4tdddt|tr=|dkrCtdd dt|trL|dkrRtdd dd at|}zGz|||IdHt d d |diWWdaSt y}z tdd|dd|d}~wt y}z t d|tddd|d}~wwdaw)NizInvalid JSON formatrnew_data_parallel_size drain_timeoutxz"new_data_parallel_size is requiredrz1new_data_parallel_size must be a positive integerz(drain_timeout must be a positive integerTr3z Scaled to z data parallel enginesFiz0Scale failed due to request drain timeout after z secondszScale failed: %siz Scale failed)jsonJSONDecodeErrorrrrr\_scaling_elastic_epr{scale_elastic_eprrrrr)rbodyr!rarbclientryryr|rgsj     rgz/is_scaling_elastic_epcstdtiS)Nis_scaling_elastic_ep)rrfr ryryr|rjVs rjINVOCATION_TYPEScCs&g|]\}\}}t|||ffqSry)pydanticZ TypeAdapter).0Z request_type get_handlerendpointryryr| ms rpz /invocationsc sz IdH}Wntjy#}z ttjjd|d|d}~wwfddtD}|D]\}}z||}Wn tj yDYq/w||IdHSfdd|D}d|}t j |d} t | | jjd S) z9For SageMaker, routes requests based on the request type.NzJSON decode error: rcs(g|]\}\}}|dur||fqSrury)rm validatorrnror ryr|rps  zinvocations..cs.g|]\}}t|jtrjntqSry)r_typetyperr)rmrq_)tryr|rpsz;Cannot find suitable handler for request. Expected one of: r2r)rdrerrrBrINVOCATION_VALIDATORSZvalidate_pythonrlValidationErrorrr5rrrr) rrhr!Zvalid_endpointsZrequest_validatorrorZ type_namesmsgresry)rrur| invocationsss:   rzz\Torch Profiler is enabled in the API server. This should ONLY be used for local development!z/start_profilec2tdt|IdHtdtddS)NzStarting profiler...zProfiler started.rr)rrr{ start_profilerr ryryr|r|    r|z /stop_profilecr{)NzStopping profiler...zProfiler stopped.rr)rrr{ stop_profilerr ryryr|r~r}r~znLoRA dynamic loading & unloading is enabled in the API server. This should ONLY be used for local development!z/v1/load_lora_adaptercDt|}||IdH}t|trt||jjdStd|dSNrr)rr ) rload_lora_adapterrr>rrrrrrrrr=ryryr|r  rz/v1/unload_lora_adaptercrr) runload_lora_adapterrr>rrrrrrryryr|rrrlog_config_filec Csx|sdSzt|}t|WdWS1swYWdSty;}ztd||WYd}~dSd}~ww)Nz0Failed to load log config from file %s: error %s)openrdloadrrr)rfr!ryryr|load_log_configs (rc @sHeZdZdZdedeeddfddZded e d e de dfd d Z dS) AuthenticationMiddlewareaK Pure ASGI middleware that authenticates each request by checking if the Authorization header exists and equals "Bearer {api_key}". Notes ----- There are two cases in which authentication is skipped: 1. The HTTP method is OPTIONS. 2. The request path doesn't start with /v1 (e.g. /health). rttokensrNcCs||_dd|D|_dS)NcSsh|]}d|qS)zBearer ry)rmtokenryryr| sz4AuthenticationMiddleware.__init__..)rt api_tokens)selfrtrryryr|__init__sz!AuthenticationMiddleware.__init__scopereceivesendcCs|ddvs |ddkr||||S|dd}t|dj|}t|d}|drB|d |jvrBtd d id d }||||S||||S)NrshttpZ websocketmethodOPTIONS root_pathrrz/v1 Authorizationr Unauthorizedir) rtrrr removeprefixr startswithrr)rrrrrZurl_pathrr=ryryr|__call__s     z!AuthenticationMiddleware.__call__) rrr__doc__r listrrr#r"r$rrryryryr|rs rc @@eZdZdZdeddfddZdeded ede dfd d Z dS) XRequestIdMiddlewarez Middleware the set's the X-Request-Id header for each response to a random uuid4 (hex) value if the header isn't already present in the request, otherwise use the provided request id. rtrNcC ||_dSrurtrrtryryr|rrzXRequestIdMiddleware.__init__rrrcsJ|ddvr |||St|ddtddffdd }||||S)Nrsrrr3rcsL|ddkrt|dd}dtj}|d||IdHdS)zx Custom send function to mutate the response headers and append X-Request-Id to it. rszhttp.response.startr)rawz X-Request-IdN)rruuiduuid4hexr)r3response_headersZ request_idZrequest_headersrryr|send_with_request_ids  z;XRequestIdMiddleware.__call__..send_with_request_id)rtrr!)rrrrrryrr|r s   zXRequestIdMiddleware.__call__ rrrrr rr#r"r$rrryryryr|rs rc @r) ScalingMiddlewarez Middleware that checks if the model is currently scaling and returns a 503 Service Unavailable response if it is. This middleware applies to all HTTP requests and prevents processing when the model is in a scaling state. rtrNcCrrurrryryr|r.rzScalingMiddleware.__init__rrrcCsH|ddkr ||||Strtddidd}||||S||||S)Nrsrrz7The model is currently scaling. Please try again later.ir)rtrfr)rrrrr=ryryr|r1s  zScalingMiddleware.__call__rryryryr|r%s r chunk_datacCszQddlm}m}|ddkr-||}|jr'|jdjjr*|jdjjWSWd SWd S|ddkrI||}|jrL|jdjrO|jdjWSWd SWd SWd St j yd|vr|dr|dd}d|vrz|ddrz|ddYS|d r|d YSYd SYd SYd Sw) z0Extract content from a streaming response chunk.r)ChatCompletionStreamResponseCompletionStreamResponseobjectzchat.completion.chunkZtext_completionchoicesdeltar textr) vllm.entrypoints.openai.protocolrrrZmodel_validaterrr rrlrw)rrrZ chat_responseZcompletion_responsechoiceryryr|_extract_content_from_chunkDsT      rc@s`eZdZdZddZdedeefddZdede fd d Z d e dd fd dZ de fddZ d S) SSEDecoderz:Robust Server-Sent Events decoder for streaming responses.cCsd|_g|_dS)Nr)buffercontent_bufferrryryr|rds zSSEDecoder.__init__chunkrcCsddl}z|d}Wn tygYSw|j|7_g}d|jvrn|jdd\}|_|d}|dri|dd}|d krM|d d in|riz| |}|d |d Wn |j yhYq wd|jvs%|S)z4Decode a chunk of SSE data and return parsed events.rNzutf-8 r zdata: z[DONE]rsdonedata)rsr) rddecodeUnicodeDecodeErrorrrrstriprstriprloadsre)rrrdZ chunk_streventslineZdata_str event_dataryryr| decode_chunkhs0      zSSEDecoder.decode_chunkrcCst|S)z Extract content from event data.)r)rrryryr|extract_contentrzSSEDecoder.extract_contentr NcCs|r |j|dSdS)zAdd content to the buffer.N)rr)rr ryryr| add_contentszSSEDecoder.add_contentcCs d|jS)z"Get the complete buffered content.r)rrrryryr|get_complete_contents zSSEDecoder.get_complete_content) rrrrrbytesrdictrrrrrryryryr|ras r response_bodycsFddlm}tdfdd}|||_tdtdS)z/Log streaming response with robust SSE parsing.rrc3sD]V}d7|V|}|D]E}|ddkr(|d}|q|ddkrX}|rNt|dkrC|ddd} td|dStddSqqdS) Nrrsrrirz;response_body={streaming_complete: content='%s', chunks=%d}z9response_body={streaming_complete: no_content, chunks=%d})rrrrlenrr)rreventr Z full_contentZ chunk_countrZ sse_decoderryr|buffered_iterators8      z2_log_streaming_response..buffered_iteratorz,response_body={streaming_started: chunks=%d}N)starlette.concurrencyrr body_iteratorrrr)r=rrrryrr|_log_streaming_responses  !rcCs>z|d}td|WdStytdYdSw)zLog non-streaming response.rzresponse_body={%s}zresponse_body={}N)rrrr)rZ decoded_bodyryryr|_log_non_streaming_responses   rc Csj|jr tdddtd}nttd}|t|j|_t||jt|j |j |j |j d| tdtdtfdd}| tdtdtfdd }d d |jpQtjgD}r]|jt|d |jre|t|ttjr~td |ddtfdd}|jD]1}|dd\}}tt||} t !| r|| qt "| r|d| qt#d|d|S)N)Z openapi_urlZdocs_urlZ redoc_urlr)r)Z allow_originsallow_credentialsZ allow_methodsZ allow_headersrtexccs4tt|jt|jj|jdd}t||jdS)Nr3rsrrr)r>r=rrrphraserr)rtrerrryryr|http_exception_handlers z)build_app..http_exception_handlercsjt|}t|}|r|r||kr|d|}n|}tt|tjjtjdd}t|tjdS)N rrr) rrr>r=rrBrrr)rtrZexc_strZ errors_strr3rryryr|validation_exception_handlers  z/build_app..validation_exception_handlercSsg|]}|r|qSryry)rmkeyryryr|rpszbuild_app..)rz}CAUTION: Enabling log response in the API Server. This can include sensitive information and should be avoided in production.rrcsz||IdH}dd|j2IdH}tt||_|jdd}|dk}|s.td|S|r7t|||St||S)Ncsg|z3dHW}|q6Srury)rmsectionryryr|rp sz3build_app..log_response..rrz text/event-stream; charset=utf-8zresponse_body={}) rriterrrrrrr)rZ call_nextr=rrZ is_streamingryryr| log_response s    zbuild_app..log_response.rzInvalid middleware z . Must be a function or a class.)$Zdisable_fastapi_docsrrZinclude_routerr(rrZadd_middlewarerZallowed_originsrallowed_methodsZallowed_headersZexception_handlerrrrZapi_keyrxZ VLLM_API_KEYrZenable_request_id_headersrrZ"VLLM_DEBUG_LOG_API_SERVER_RESPONSErr middlewarersplitr' importlib import_moduleinspectisclassiscoroutinefunction ValueError) rrtrrrrr module_pathZ object_nameZimportedryryr| build_appsR          rrr~cshjdur j}njg}jrtjd}nd}fdd|D}||_j |_||_|j }t j r<| IdH}n|j }td|tj} | duru|IdH} t| tr`t| d} nt| dd|j d} | | krutd| jjdkr~t} njrt} | jIdHnd} |jdur|jjni} j}| rd d| D}jdur|}n||7}t ||||d |_!|j!"IdHd |vrt#|||j!|| j$j%j&j'| j(j)j*d nd|_+d |vrt,|||j!j-|| j$j%j&j.j'j(j)j*d nd|_/d |vrt0|||j!|j%j)j*dnd|_1d|vr2t2|||j!|| j$dnd|_3d|vrFt4|||j!|| j$dnd|_5d|vrWt6|||j!|dnd|_7d|vogt8|j9dddk}d|vsp|ryt:|||j!|dnd|_;t<|||j!|| j$d|_=d|vrt>|||j!|dnd|_?d|vrt@|||j!|dnd|_AjB|_Bd|_CdS)N) max_log_lencsg|] }t|jdqS))rZ model_path)rTr)rmrrryr|rp;s z"init_app_state..zSupported_tasks: %s) chat_template) tokenizerrZtools model_configzUsing supplied chat template: %s It is different from official chat template '%s'. This discrepancy may lead to performance degradation.democSsg|] \}}t||dqS))rr)rU)rmZmodalityZ lora_pathryryr|rpss)r{rbase_model_paths lora_modulesgenerate) request_loggerrchat_template_content_formatreturn_tokens_as_token_idsenable_auto_tools tool_parser tool_serverreasoning_parserenable_prompt_tokens_detailsenable_force_include_usage) rrrrr#exclude_tools_when_tool_choice_nonerrrr)rrrrencode)rrrZembedr)rZ num_labelsrrr)DZserved_model_namerrr0rr{rrrrrxrZget_supported_taskssupported_tasksrrr,rZ get_tokenizerrrhr.r-rrr^r_Zadd_tool_serverZ lora_configdefault_mm_lorasritemsrVrZinit_static_lorasrXrrenable_auto_tool_choicetool_call_parserrrrrrOZ response_rolerrrQrrWrrRrrPrr'Z hf_configrYrrZrr[rr\rZenable_server_load_trackingr )r{rr~rZserved_model_namesrrrrZresolved_chat_templaterZhf_chat_templaterrrZdefault_mm_lora_pathsZenable_serving_rerankingryrr|init_app_state+st                   r addrcCsZtj}t|dr tj}tj|tjd}|tjtjd|tjtjd| ||S)Nrfamilyrsr) socketAF_INETrnAF_INET6 SOCK_STREAM setsockopt SOL_SOCKET SO_REUSEADDR SO_REUSEPORTbind)r rsockryryr|create_server_sockets  rrcCs tjtjtjd}|||S)Nr)rAF_UNIXrr)rrryryr|create_server_unix_sockets rcCsxtj}|jr|j|vrtd|jdd|dtj}|j r8|j |vr:td|j dd|ddSdS)Nzinvalid tool call parser: z (chose from { ,z })zinvalid reasoning parser: ) r]Z tool_parserskeysr r KeyErrorrrfZreasoning_parsersr)rZvalid_tool_parsesZvalid_reasoning_parsesryryr|validate_api_server_argss      r c Cstdtt||jrt|jdkrt|jt||j r't |j }n |j p+d|j f}t |}tddd}ttj||j rOd|j }||fS|\}}|joX|j}t|rcd |d n|pfd }d |rld ndd|d|}||fS)zRValidate API server args, set up signal handler, create socket ready to serve.zvLLM API server version %srrNcWstd)NZ terminated)KeyboardInterrupt)rtryryr|signal_handler"rz$setup_server..signal_handlerzunix:[]z0.0.0.0rsz://:rN)rrr,rctool_parser_pluginrr]import_tool_parserr ZudsrhostportrrosignalSIGTERM ssl_keyfile ssl_certfilern) rrZ sock_addrr#listen_addressr r,Zis_sslZ host_partryryr| setup_server s0      r2cs4tdt|\}}t|||fi|IdHdS)zRun a single-worker API server.Z APIServerN)rlr2run_server_worker)ruvicorn_kwargsr1rryryr| run_server3s r5c sB|jrt|jdkrt|j|r|ddnd}t|j}|dur(||d<t||d4IdHR}t|t |}| IdH} t || |j |IdHt d||t|f||j|j|j|j|j tj|j|j|j|j|j|jd |IdH} WdIdHn 1IdHswYz | IdHW|dS|w) zRun a single API server worker.r!rrN log_config)rz!Starting vLLM API server %d on %s) renable_ssl_refreshr+r, log_levelZ access_logZtimeout_keep_aliver/r0 ssl_ca_certs ssl_cert_reqsh11_max_incomplete_event_sizeh11_max_header_count)r)rr]r*rrrrr)rZget_vllm_configr r~rrr/r7r+r,Zuvicorn_log_levelZdisable_uvicorn_access_logrxZVLLM_HTTP_TIMEOUT_KEEP_ALIVEr/r0r9r:r;r<r) r1rrrr4Z server_indexr6r{rtrZ shutdown_taskryryr|r3=sT  (! r3__main__z*vLLM OpenAI-Compatible RESTful API server.) descriptionr(ru(%rvrrrrrdrZmultiprocessing.forkserverrrr-rrrargparsercollections.abcrr contextlibr functoolsrrrtypingrr r r rrlregexrZuvloopZfastapir r rrrrZfastapi.exceptionsrZfastapi.middleware.corsrZfastapi.responsesrrrrZ!prometheus_fastapi_instrumentatorrrrZstarlette.datastructuresrrrrZstarlette.routingrZstarlette.typesr r!r"r#r$Ztyping_extensionsr%Z vllm.envsrxZ vllm.configr&Zvllm.engine.arg_utilsr'Zvllm.engine.async_llm_enginer(Z"vllm.engine.multiprocessing.clientr)Z"vllm.engine.multiprocessing.enginer*Zvllm.engine.protocolr+Zvllm.entrypoints.chat_utilsr,r-r.Zvllm.entrypoints.launcherr/Zvllm.entrypoints.loggerr0Z vllm.entrypoints.openai.cli_argsr1r2rr3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrLrMrNZ$vllm.entrypoints.openai.serving_chatrOZ.vllm.entrypoints.openai.serving_classificationrPZ*vllm.entrypoints.openai.serving_completionrQZ)vllm.entrypoints.openai.serving_embeddingrRZ&vllm.entrypoints.openai.serving_enginerSZ&vllm.entrypoints.openai.serving_modelsrTrUrVZ'vllm.entrypoints.openai.serving_poolingrWZ)vllm.entrypoints.openai.serving_responsesrXZ%vllm.entrypoints.openai.serving_scorerYZ,vllm.entrypoints.openai.serving_tokenizationrZZ-vllm.entrypoints.openai.serving_transcriptionr[r\Z$vllm.entrypoints.openai.tool_parsersr]Zvllm.entrypoints.tool_serverr^r_r`Zvllm.entrypoints.utilsrarbrcrdZ vllm.loggerreZvllm.reasoningrfZvllm.transformers_utils.configrgZ!vllm.transformers_utils.tokenizerrhZvllm.usage.usage_libriZ vllm.utilsrjrkrlrmrnroZvllm.v1.metrics.prometheusrpZ vllm.versionrqr,r__annotations__rsetrsTaskrrrrrrrrr(rrrrrrrrrrrrrrrr{rr rpostrrBr NOT_FOUNDrrr"r$r)r*r.OKr6r;r?r@rArDrErGrHrKUNPROCESSABLE_ENTITYrOrQrRrSrTZVLLM_SERVER_DEV_MODErrUrYrwr_r`REQUEST_TIMEOUTrgrjZ RequestTypeZ GetHandlerFnZ EndpointFnrkrtuplervUNSUPPORTED_MEDIA_TYPErzZVLLM_TORCH_PROFILER_DIRr|r~Z VLLM_ALLOW_RUNTIME_LORA_UPDATINGrrrrrrfrrrrrrr r\rrr r2r5r3rparser parse_argsrrunryryryr|s                       x                    "                   .       ,      #"5- _ "= * 8