o )i+ @sddlZddlZddlZddlZddlmZddlmZddlmZddl m Z m Z m Z m Z mZmZmZmZddlZddlmZddlmZmZmZddlmZmZmZmZe rfdd lmZdd l m!Z!m"Z"ee#Z$ed Z%Gd d d e e%eZ& d(de'de(de)de(fddZ*GdddZ+  d)de+deeddedddfddZ,de-efddZ.d ej/d!ej/d"e)dej/fd#d$Z0ej1fd%eddfd&d'Z2dS)*N)Sequence) connection) BaseProcess) TYPE_CHECKINGAnyCallableGenericOptionalTypeVarUnionoverload) init_logger) UsageContextis_usage_stats_enabled usage_message) get_open_portget_open_zmq_ipc_path get_tcp_urikill_process_tree) DPCoordinatorCoreEngineActorManagerCoreEngineProcManagerTc @sFeZdZdeeddfddZddZdd Zd d Zd d Z ddZ ddZ  d,dede de e de fddZede defddZededeefddZdee efdeeeeffddZede defddZededefd dZdee efdeeeeffd!dZd"d#Zd$d%Zd&d'Zd(d)Zd*d+ZdS)- ConstantListxreturnNcCs ||_dSN_x)selfrr!Y/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/v1/utils.py__init__! zConstantList.__init__cCtd)Nz Cannot append to a constant list TypeErrorr itemr!r!r"append$zConstantList.appendcCr%)NzCannot extend a constant listr&r(r!r!r"extend'r+zConstantList.extendcCr%)Nz"Cannot insert into a constant listr&r(r!r!r"insert*r+zConstantList.insertcCr%)NzCannot pop from a constant listr&r(r!r!r"pop-r+zConstantList.popcCr%)Nz"Cannot remove from a constant listr&r(r!r!r"remove0r+zConstantList.removecCr%)NzCannot clear a constant listr&r r!r!r"clear3r+zConstantList.clearrr)startstopcCs$|j|||dur |St|jSr)rindexlen)r r)r2r3r!r!r"r46s  zConstantList.indexcCdSrr!r(r!r!r" __getitem__=zConstantList.__getitem__scCr6rr!)r r9r!r!r"r7Ar8cCs |j|Srrr(r!r!r"r7Er$valuecCr6rr!r r)r:r!r!r" __setitem__Hr8zConstantList.__setitem__cCr6rr!)r r9r:r!r!r"r<Lr8cCr%)Nz"Cannot set item in a constant listr&r;r!r!r"r<Pr+cCr%)Nz'Cannot delete item from a constant listr&r(r!r!r" __delitem__Sr+zConstantList.__delitem__cC t|jSr)iterrr0r!r!r"__iter__Vr$zConstantList.__iter__cCs ||jvSrrr(r!r!r" __contains__Yr$zConstantList.__contains__cCr>r)r5rr0r!r!r"__len__\r$zConstantList.__len__cCsd|jdS)Nz ConstantList()rr0r!r!r"__repr___szConstantList.__repr__)rN)__name__ __module__ __qualname__listrr#r*r,r-r.r/r1intr r4r r7slicer r<r=r@rArBrDr!r!r!r"rsD && r local_onlyhostportrcCs|rtSt||p tS)a Assign a new ZMQ socket address. If local_only is True, participants are colocated and so a unique IPC address will be returned. Otherwise, the provided host and port will be used to construct a TCP address (port == 0 means assign an available port).)rrr)rKrLrMr!r!r"get_engine_client_zmq_addrcs rNc@sVeZdZdZ ddedededejde de ed e ed e efd d Z dddZ dS)APIServerProcessManagerzManages a group of API server processes. Handles creation, monitoring, and termination of API server worker processes. Also monitors extra processes to check if they are healthy. Ntarget_server_fnlisten_addresssockargs num_serversinput_addressesoutput_addressesstats_update_addressc Cs||_||_||_td} g|_tt|||D]-\} } } | | || d} |dur-|| d<| j|d| |||| fd}|j || qt dt |jt|t|j|_dS)a@Initialize and start API server worker processes. Args: target_server_fn: Function to call for each API server process listen_address: Address to listen for client connections sock: Socket for client connections args: Command line arguments num_servers: Number of API server processes to start input_addresses: Input addresses for each API server output_addresses: Output addresses for each API server stats_update_address: Optional stats update address Zspawn)Z input_addressZoutput_addressZ client_countZ client_indexNrWZ ApiServer_)targetnamerSzStarted %d API server processes)rQrRrSmultiprocessingZ get_context processesziprangeProcessr*r2loggerinfor5weakreffinalizeshutdown _finalizer)r rPrQrRrSrTrUrVrWZ spawn_contextiZin_addrZout_addrZ client_configprocr!r!r"r#ys2    z APIServerProcessManager.__init__rcCs |dSr)rdr0r!r!r"closes zAPIServerProcessManager.closer)rN)rErFrG__doc__rstrrargparse NamespacerIrHr r#rgr!r!r!r"rOrs* 7rOapi_server_managerengine_manager)rr coordinatorrc Csddlm}m}zzqtddd|jD}|r |j||jj<g}t||r3|jD]}|||j<q*n t||r<| }|s@|ryt j |dd}|D]} | | }|j dkretd|jd |jd |j qI|rudd l} | j |dd\} }|s@|s@Wn"tytd Ynty} z td t| d } ~ wwWtd||r||r|d Sd Std||r||r|ww)aWait for all processes to complete or detect if any fail. Raises an exception if any process exits with a non-zero status. Args: api_server_manager: The manager for API servers. engine_manager: The manager for engine processes. If CoreEngineProcManager, it manages local engines; if CoreEngineActorManager, it manages all engines. coordinator: The coordinator for data parallel. rrz'Waiting for API servers to complete ...cSsi|]}|j|qSr!)sentinel).0rfr!r!r" sz2wait_for_completion_or_failure..)timeoutzProcess z (PID: z) died with exit code Nz8Received KeyboardInterrupt, shutting down API servers...z0Exception occurred while running API servers: %sz#Terminating remaining processes ...)vllm.v1.engine.utilsrrr_r`r[rfro isinstanceZ get_run_refsrwaitr.exitcode RuntimeErrorrYpidrayKeyboardInterrupt Exception exceptionrirg) rlrmrnrrZsentinel_to_procZactor_run_refsrfZready_sentinelsrorz_er!r!r"wait_for_completion_or_failuresr            rprocscCs|D] }|r |qtd}|D]}|t}|dkr#n |r,||q|D]}|r@|j}dur@t|q/dS)Nrrr)is_alive terminatetime monotonicjoinryr)rrfdeadline remainingryr!r!r"rcs"   rc from_tensor to_tensorlengthcCs|d|j|d|ddS)z Copy the first length elements of a tensor into another tensor in a non-blocking manner. Used to copy pinned CPU tensor data to pre-allocated GPU tensors. Returns the sliced target tensor. NT)Z non_blocking)Zcopy_)rrrr!r!r" copy_slices r usage_contextcCsxtsdSddlm}tj||j|t|jj|jj |j j |j j |jj t|j jt|j|j j|jj|jjd ddS)z#Report usage statistics if enabled.Nr)get_architecture_class_name) dtypetensor_parallel_size block_sizegpu_memory_utilization quantizationZkv_cache_dtypeZ enable_loraenable_prefix_caching enforce_eagerdisable_custom_all_reduce)Z extra_kvs)rZ vllm.model_executor.model_loaderrrZ report_usageZ model_configrirZparallel_configrZ cache_configrrrZ cache_dtypeboolZ lora_configrrr)Z vllm_configrrr!r!r"report_usage_stats!s$    r)r)NN)3rjrZrracollections.abcrrZmultiprocessing.processrtypingrrrrr r r r ZtorchZ vllm.loggerr Zvllm.usage.usage_librrrZ vllm.utilsrrrrZvllm.v1.engine.coordinatorrrtrrrEr_rrrrirIrNrOrrHrcZTensorrZENGINE_CONTEXTrr!r!r!r"sd   (  F E  L