o ưi5@sddlZddlZddlZddlZddlZddlZddlmZmZddl Z e e Z ddl mZmZgdZddgZdeefdd Zdefd d Zd d Zd*dedeefddZdedefddZddZdefddZdededeeeffddZ   d+dedeedeedeefd d!Z!d"ed#edefd$d%Z"      d,dedeed&eedeed'eedeedeefd(d)Z#dS)-N)ListOptional)DEFAULT_HEALTH_CHECK_PROMPTHEALTH_CHECK_TIMEOUT_SECONDS)messagesZapi_keypromptinputZvertex_credentialsZaws_access_key_idZaws_secret_access_keymodelZ mode_errorreturncCsRzddl}||jj}tjdkrt|dWSt|dWSty(YdSw)zj Get process RSS memory in MB. On Linux, ru_maxrss is in KB. On macOS, ru_maxrss is in bytes. rNdarwinii)resourceZ getrusageZ RUSAGE_SELF ru_maxrsssysplatformfloat Exception)r r rQ/home/app/Keep/.python/lib/python3.10/site-packages/litellm/proxy/health_check.py_get_process_rss_mbs  rcCst}|dur dS|dS)Nunknownz.2f)r)Zrss_mbrrr_rss_mb_for_log-srcCsddg}dt|dgS)z, Get a random message from the LLM. zHey how's it going?z What's 1 + 1?user)Zrolecontent)randomchoice)rrrr_get_random_llm_message4srT endpoint_datadetailscCs8|dd|durdd|DSdd|DS)z7 Clean the endpoint data for display to users. Zlitellm_logging_objNFcSsi|] \}}|tvr||qSr)ILLEGAL_DISPLAY_PARAMS.0kvrrr Cz(_clean_endpoint_data..cSsi|] \}}|tvr||qSr)MINIMAL_DISPLAY_PARAMSrrrrr#Er$)popitems)rrrrr_clean_endpoint_data=s r( model_listcCsZt}g}|D]#}|dpi}|dpd}|durq||vr*||||q|S)N model_infoid)setgetaddappend)r)Zseen_idsZfiltered_deploymentsZ deploymentZ _model_infoZ_idrrrfilter_deployments_by_idIs  r0cs4z t||IdHWStjyddiYSw)NerrorzTimeout exceeded)asynciowait_for TimeoutError)tasktimeoutrrrrun_with_timeout\s  r7csZ|d}|di}|dd}t||}|dpt}ttj||tdgd|IdHS)Nlitellm_paramsr*modeZhealth_check_timeoutztest from litellm)r9rr)r-'_update_litellm_params_for_health_checkrr7litellmZ ahealth_checkr)r r8r*r9r6rrr_run_model_health_checkes    r<modelsconcurrency_limitc sdgt|}itt|ddtffdd }tt|t|D]}|q&rmtjt tj dIdH\}}|D])} |}z| ||<Wnt yf}z |||<WYd}~nd}~ww|qAs.|fS)z{ Run health checks with at most `concurrency_limit` active tasks. Preserves result ordering to match `models`. Nrr csNzt\}}Wn tyYdSwtt|}||<ttdS)NFT)next StopIterationr2 create_taskr<maxlen)idxZ next_modelr5Z model_iterpeak_in_flightZtasks_to_indexrr_schedule_nexts zC_run_health_checks_with_bounded_concurrency.._schedule_next) return_when)rCiter enumerateboolrangeminr2waitr,keysFIRST_COMPLETEDr&resultr) r=r>resultsrG_doner5rDerrEr+_run_health_checks_with_bounded_concurrencyws0     rVmax_concurrencyinstrumentation_contextc sF|pi}t|dd}|dd}|dd}d}d}t|tr3|dkr3d}t||Id H\} }nd d |D} t| }tj| d d iId H} |r]t d|||t|||t t g} g} t | |D]8\} }|d}t| trd| vr| ti|| |qft| tr| ti|| |qf| t||qf| | fS)z Perform a health check for each model in the list. max_concurrency: Optional limit on concurrent health check requests. enabledFcycle_idrsourceZ unboundedrZboundedNcSsg|] }tt|qSr)r2rAr<)r r rrr sz)_perform_health_check..return_exceptionsTzhealth_check_dispatch_summary source=%s cycle_id=%s mode=%s model_count=%d max_concurrency=%s peak_in_flight=%d thread_count=%d rss_mb=%sr8r1)rKr- isinstanceintrVrCr2gatherloggerdebug threading active_countrzipdictr/r()r)rrWrXinstrumentation_enabledrZr[Z dispatch_moderFrRtaskshealthy_endpointsunhealthy_endpointsZ is_healthyr r8rrr_perform_health_checksV     rkr*r8c Cst|d<|dd}|dur||d<nd|dp |dp dvr&d |d<|dd}|dur4||d<|d dd krD|d d |d<|ddrddlm}|d}|dr`|dd}|d}g}|D] }||jvru||qid|}||d<|S)a Update the litellm params for health check. - gets a short `messages` param for health check - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID rZhealth_check_max_tokensNZ max_tokens*Zhealth_check_modelr r9Z audio_speechZhealth_check_voicealloyZvoicezbedrock/r)BedrockModelInfo/) rr- startswithZ!litellm.llms.bedrock.common_utilsrpsplitZall_global_regionsr/join) r*r8Z_health_check_max_tokensZ_health_check_modelrpr partsZfiltered_partspartrrrr:s6          r: cli_modelmodel_idc s|pi}t|dd}|dd}|dd} |s3|r&|d|idg}n |r/td| |ggfSt} t|} d urMfd d |D} | rL| }nd urifd d |D} | gkrgfd d |D} | }t|}t|d}t|}|rtd| || |||t t zt ||||dId H\}}Wnt y|rt d| ||t| dt t w|rtd| ||t|t|t| dt t ||fS)av Perform a health check on the system. When model_id is provided, only the deployment with that id is checked (so models that share the same name but have different ids are checked separately). When model (name) is provided, all deployments matching that name are checked. Returns: (bool): True if the health check passes, False otherwise. rYFrZrr[r ) model_namer8zAhealth_check_cycle_skipped source=%s cycle_id=%s reason=no_modelsNcs(g|]}|dp idkr|qS)r*r+)r-r x)ryrrr\Ls"z(perform_health_check..cs g|] }|ddkr|qS)r8r rr{r rrr\Rscsg|] }|dkr|qS)rzrr{r}rrr\Vs)r)zhealth_check_cycle_start source=%s cycle_id=%s requested_model_count=%d post_model_filter_count=%d deduped_model_count=%d max_concurrency=%s thread_count=%d rss_mb=%s)rWrXzihealth_check_cycle_failed source=%s cycle_id=%s model_count=%d duration_ms=%.2f thread_count=%d rss_mb=%sizhealth_check_cycle_complete source=%s cycle_id=%s model_count=%d healthy_count=%d unhealthy_count=%d duration_ms=%.2f thread_count=%d rss_mb=%s)rKr-rarbtime monotonicrCr0rcrdrrkr exception)r)r rxrryrWrXrgrZr[Zcycle_start_timeZrequested_model_countZ_by_idZ_new_model_listZpost_filter_model_countZdeduped_model_countrirjr)r ryrperform_health_check!s          r)T)TNN)NNTNNN)$r2loggingrrrcr~typingrrr; getLogger__name__raZlitellm.constantsrrrr%rrstrrrrfrKr(r0r7r<listr_tuplerVrkr:rrrrrs        , @ B