o ưi@s?UddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlmZmZmZddlmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!ddl"Z"ddl#Z#ddl$Z#ddl%m&Z&m'Z'ddl(m)Z)ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9ddl:m;Z;ddlm?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]dd l^m_Z_m`Z`dd lambZbdd lcmdZdmeZemfZfmgZgdd lhmiZimjZjer3ddlkmlZlddlmmnZoddlpmqZqeeoefZnneZneZqddddZreesefetd<dddZueue_uejvdewdgZxeyetd<e jz{dejz|dzddl}Z}ddl~Z~ddlZddlZddlZddlmZWneyZzededdZ[wwgdZd d!Zdd"lmZdd#lmZdd$lmZddlZdd%lmZdd&lmZmZdd'lmZmZdd(lmZdd)l*mZmZmZmZmZmZmZmZmZmZmZmZmZmZdd*lmZdd+lmZdd,lmZdd-lmZdd.lmZmZdd/lmZdd0l:mZdd1lmZdd2lmZmZdd3lmZdd4lmZdd4lmZdd4lmZdd5lmZdd6lmZdd7l>Tdd4lmZdd8lmZdd4lmZdd9lmZmZdd4lmZdd:lmZdd4lmZdd4lmZdd;lmZmZmZddlmZdd?lmZmZmZmZmZdd@lmZmZmZdd4lmZdd4lmZddAlmZmZddBl^mZddClmZdd4lmZddDlmZmZddElmZddFlmZmZmZmZddGlmZmZddHlmZddIlmZddJl m Z ddKl m Z dd4l mZdd4lmZddLlmZddMlmZddNlmZdd4lmZddOlmZdd4lmZdd4lmZddPlmZm Z ddQl!m"Z"dd4l#mZ$ddRl%m&Z&ddSl'm(Z(ddTl)m*Z*dd4l+mZ,ddUl-m.Z.dd4l/mZ0dd4l1mZ2dd4l3mZ4dd4l5mZ6dd4l7mZ8ddVl9m:Z:m;Z;dd4lmZ?dd4l@mZAdd4lBmZCdd4lDmZEddWlDmFZFdd4lGmZHddXlImJZJmKZKmLZLdd4lImZMdd4lNmZOdd4lPmZQddYlRmSZSmTZTmUZUdd4lRmZVdd4lWmZXdd4lYmZZdd4l[mZ\dd4l]mZ^ddZl_m`Z`dd4lamZbdd4lcmZddd4lemZfdd[lemgZgmhZhdd4limZjdd\lkmlZldd4lkmZmdd4lnmZodd4lpmZqdd]lrmsZsdd^ltmuZudd_lvmwZwdd4lxmZydd4lzmZ{dd4l|mZ}dd`l|m~Z~ddalmZdd4lmZddblmZddclmZdd4lmZdd4lmZdd4lmZdd4lmZdd4lmZdd4lmZdd4lmZdd4lmZdddlmZdd4lmZdd4lmZdd4lmZdd4lmZddelmZddflmZdd4lmZddglmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZdd4lmZdd4lmZdd4lmZdd4lmZdd4lmZddhlmZmZmZmÐZddilĐmŐZŐmƐZddjlǐmȐZddklɐmʐZddllːm̐Z̐m͐Z͐mΐZΐmϐZddmlАmѐZddnlҐmӐZӐmԐZԐmՐZՐm֐ZddolאmؐZddplِmڐZddqlېmܐZܐmݐZddrlސmߐZddslmZddtlmZddulmZmZmZddvlmZddwlmZmZddxlcmZmZddtlcmZddylcmZmZddzlhmZz dd{lmZWneyDd|ZYnwd}e_ddlZdd~lmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZddlm Z ddl m Z ddl m Z ddlmZmZmZmZmZddlmZddlmZddlmZddlmZddlmZeZzddlm m!Z!Wney2zddl!Z!Wn ey/YnwYnwz#dd4l"mZ#ddl$m%Z%e#Ze%Z&ee%etd<Wn eycdZ&YnweZ'ea(t()a*e+etd<t(j,Z-edetd<e.dZ/eesetd<eZ0eZ1e/durdZ2e3etd<ne3e/Z2e.dZ4eesetd<e4durdZ5e6etd<ne6e4Z5e'dZ7e'dZ8e'dZ9de7de8dZ:e:d7Z:e:de9d7Z:e'dZ;e:de;d7Z:dZ<t* r&e.ddndZ=t* rddZ?ddZ@ddZAedefddZBeeee=e>ee'eBdZeZCddlmDZDddZEddZFe.dddk rt* reFeĐ_GneEeĐ_GGdddejHZIeĐJeVdedeVfddZKeZdgZLzejzMejz|eNZOejzPeOddġZQeQZRdŐZSdesdǐe+fddɄZTdesdǐe+fdd̄ZUdesdǐe+fddτZVdesdesdǐeWe+esffddӄZXe.ddաYdkZZeZ r3dאZ[neQZ[e.dؐe[Z\eUeQ sRe]dِeQdڝe\eQkZ^e^ rҐeVe\Z_eTe\Z`e` re_ readېe\e\ZRn^e` re_ sebdܐe\dݝe\ZRnFeadސe\dߝeXeQe\\ZcZ]ec re\ZRn&ebde\de]deQdeQZRneadeQeQZReUeR se]deRde' re'dk redeRejeZfef sebdeRdnegeRD]\ZhZiZjejD]ZkejzPehekZlekmd rA q)zbeneldddZoeopZqWdn 1 s_wYeqreSe'Zsesrde'dZseneldddZoeotesWdn 1 swYW q)euevewf yY q)w qeĐjxdeejzPeRddddeĐjxeSdeejzPeRddddeĐjxdeeRd}ddddesddfddZyzDeVeRZ_edeRejeZfe_ readeRdn ef s/ebdeRdneyeReadeRWnCev ybZzezdeRdeWYdZ[n&dZ[we yZzezdeRdeWYdZ[ndZ[wwWn e yYnwejzMejz|eNZOeĐj{e eLd}dgdge8deĐ{eweĐ{euddZ|e|eZ}e.dZ~eesetd<e}dk re~du reĐjdddd d Zdd lmZdadadadadadad}adZdadaeesetd <d}Zdadaeeetd <daeeyetd<iaeetd<daeeeesefetd<dZdZdaeesetd<dZeeeetd<dadaeeetd<daedetd<eeIjjdae&tdZejedaeeetd<daeedeese+fetd<gaeesetd<dae3etd<dadadadadadadadadadaiaeesee3eeesefffetd<dadagZeetd <d!Zeadaeed"efetd#<eaeaeaeadadaeZdaee(etd$<dae+etd%<daeeqetd&<ett*d'adZdZdZdadadadaeeetd(<defd)d*ZƐd+d,Zdefd-d.ZȐdؐd/e+fd0d1Zɐd2d3Z dِd4eesd5eesd6eesd7eesd8ee6d9eend:eeesfd;d<Zːd=d>Zdee6fd?d@ZdesfdAdBZΐdCeydDee+dEee3dFefdGdHZϐdCeydIeydJeyfdKdLZАdMdNZGdOdPdPeZGdQdRdRZӐeӃaԐdSdTZ         U   } }    dڐdVdWZ֐dXdYZאdZe]d[efd\d]Zؐd[edesfd^d_Zِd`edaesd[edbe+deee+ff dcddZڐdZe]d[efdedfZېdZe]d[efdgdhZifdiefdjdkZݐdldmZސdndoZGdpdqdqZejdreegdsgdtejdueegdsgdteedddddddfdZe]dvee+d7eesdwee+dxee+dyee+dzeesd{eesfd|dCZejd}eegdsgdtejd~eegdsgdteefdesdZe]fddZejdeegdgdtejdeegdgdtejdeegdgdtejdeegdgdddiie ddeefdededieesdZe]fddZejdeegdgdtejdeegdgdtejdeegdgdtejdeegdgdtdeefdededieesdZe]fddZejdeegedgdejdeegedgdejdeegedgdejdeegedgddeefdededieesdZe]fddZejdeegedgdejdeegedgdeefdededZe]fddZdedeedffddZejdeegdgdtejdeegdgdteefdededZe]fddZejdeegdgdtejdeegdgdtedeefdedededZe]fddZeĐdejdddejdddejdddeefdedieesdeesdeesfddZeeddiesdeesdeeesesfdffddZeĐdeĐdejdddejdddeefdediesdesdeesfddZejdeegdgdtejdĐeegdgdteefdededZe]fdŐdƄZejdeegdgdtejdĐeegdgdteefdededZe]fdǐdȄZejdɐeegdgdtejdʐeegdgdteefdedesdedZe]fd̐d̈́ZejdΐeegdgdtejdϐeegdgdteefdededZe]fdАdфZejdҐeegdgdtejdӐeegdgdteefdedesdedZe]fdՐdքZejdאeegdgdtejdؐeegdgdteefdedesdedZe]fdِdڄZejdאeegdgdtejdؐeegdgdteefdedesdedZe]fdېd܄ZejdݐeegdgdtejdސeegdgdteefdedesdedZe]fdߐdZddlmZdedesdeeeeeseesffddZejddgeegegddde[de+fddZejddgeegddiesfddZejddgeegedde\fddZdeedZe]deedeefddZdeedeRdeefddZdeed edZe]deefddZdeeQd edeeseesffddZdeeesedfded edeeseesffddZdeRd edeesfddZdZe]ded edeedeef ddZ dېdieesefde+d eedeeseffdd Z dِdeeesefd esd e3d e3deed edeesdeeeesefee3ffddZdedeefddZ dܐdeeesefdeesdesdeeeseffddZ deeesefd e3d e3dee3d eesdeeseff ddZ deeesefd7esded edeeeseff ddZ desd eesdǐeWeyee3ffddZ ejdd dsgeegdd!eeejdd"dejdd#dejdd$dded%d&d%d'ed(d)d%d'ejdd*dejdd+dejdd,dejdd-dejdd.df dZe]dieesd/ee+d0ee+dee+d e3d e3d eesd1eesd2eesd3eesd4eesfd5d6Z ejd7d8dsgdeegd9eedddfdZe]d:eesd;eedZejd?d@dsgdeegd9eedAddddfdZe]d:eesd;eedt?t@t*d%IdHt(AIdHt(BIdHtCrt:;tDt(Et(FtGIdHaHdVtHdurztHIIdHtd&Wnty}ztd'|WYd}~nd}~wwt'durRtJt'd(rRtJt'jKd)rRz t'jKLIdHWntyQ}ztd*|WYd}~nd}~wwt'durtJt'd+rz t'MIdHWnty}ztd,|WYd}~nd}~wwtNIdHdS).NrZLITELLM_WORKER_STARTUP_HOOKS,rYzInvalid hook spec 'z1': expected format is 'module.path:function_name'z.Worker startup hook '%s' executed successfullyz#Worker startup hook '%s' failed: %szElitellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - {}FLITELLM_MASTER_KEY WORKER_CONFIGZCONFIG_FILE_PATHzworker_config: %sconfig_file_pathrrmLITELLM_CONFIG_BUCKET_NAMEZ DATABASE_URL) database_urlproxy_logging_objuser_api_key_cache) llm_routerrqredis_usage_cache)general_settingsrtz(About to initialize semantic tool filterlitellm_settingszlitellm_settings keys = )rsrvz)After semantic tool filter initializationzSemantic filter init failed: Texc_info)rurNrrrzprisma_client: %s)litellm_proxy_budget_name)litellm_proxy_admin_namerrrN)rurN!proxy_budget_rescheduler_min_time!proxy_budget_rescheduler_max_timeproxy_batch_write_atrqz,SESSION REUSE: Closed shared aiohttp sessionz&Error closing shared aiohttp session: dbstop_token_refresh_taskz#Error stopping token refresh task: stop_db_health_watchdog_taskz(Error stopping DB health watchdog task: rd)Ojsonrosenvirongetsplitstrip ValueErrorrsplit importlib import_modulegetattrinspectiscoroutinefunctionrxrRr\errorrSrqr9_license_check is_premiumrrCrrWisfile proxy_configis_yaml load_configrsllm_model_listru isinstancestrdict initializeloadsrNProxyStartupEvent_setup_prisma_clientrqrr_initialize_startup_loggingrt)_validate_redis_transaction_buffer_configget_config_statelistkeys _initialize_semantic_tool_filter_initialize_jwt_authprompt_injection_detection_objupdate_environmentrT max_budget_add_proxy_budget_to_dbrzasyncio create_task_warm_global_spend_cache$initialize_scheduled_background_jobsr{r|r}"_update_default_team_member_budget%_sync_ui_settings_to_general_settingsrJ_run_background_health_check_init_dd_tracer_init_pyroscoperfshared_aiohttp_sessionrWhasattrr~rrr])rrZ_startup_hooks_envZ _hook_specZ _module_pathZ _func_name_moduleZ_hook_fnre worker_configZenv_config_yamlZ_db_url_config_litellm_settingsrdrdreproxy_startup_eventsj                                 r)docs_urlZ redoc_urltitle descriptionr root_pathZlifespan)APIWebSocketRoutec CsTtjrtjSddlm}|ttjtjtjtjd}ddtjD}|D]g}|j dd d}g}z(t |drY|j durYt|j d g}|rY|D]}||jd |jd d id qHWn ttfyeYnwdd|jpl|dd|jpx|dd|dddiidgdi|d|<q#ddlm}||}trddtdig|d<|t_tjS)Nr)get_openapi_schema_with_compat)Zget_openapi_funcrrrroutescSsg|] }t|tr|qSrd)rr).0routerdrdre s  z&get_openapi_schema..{? dependant query_paramsqueryrUstring)nameinrequiredZschemarz WebSocket: zWebSocket connection endpointZ websocket_/_Z101rzWebSocket Protocol Switchedr$)summaryrZ operationId parameters responsestagspathsCustomOpenAPISpecurlservers)ropenapi_schemaZ0litellm.proxy.common_utils.openapi_schema_compatrr,rrrrrWrrstriprrrappendrrAttributeError TypeErrorreplace.litellm.proxy.common_utils.custom_openapi_specradd_llm_api_request_schema_bodyserver_root_pathr) rrZwebsocket_routesr base_pathrrparamrrdrdreget_openapi_schemas`     rcCstjrtjSt}tjj}i}|D]}||dvr!|d|||<q||d<ddlm}||}t r?ddt dig|d<|t_tjS)Nrrrrrr) rrrZ LiteLLMRoutes openai_routesvaluerrrrr)rrZpaths_to_includerrrdrdrecustom_openapi.s    rZ DOCS_FILTEREDFalseTruec@seZdZdZdS)UserAPIKeyCacheTTLEnumrjN)r\ __module__ __qualname__in_memory_cache_ttlrdrdrdrerOsrrequestexccs4|j}|}t|jrt|jntjd|i|dS)Nr) status_codecontentheaders)rto_dictr.codeintr'HTTP_500_INTERNAL_SERVER_ERROR)rrrZ error_dictrdrdreopenai_exception_handlerSsrr _experimentaloutz/litellm-asset-prefixrWreturncCs2ztj|o tt|WStyYdSw)NF)rrWisdiranyscandirFileNotFoundError)rWrdrdre_dir_has_contentls  rui_pathcCsLtj|sdStjtj|dsdStj|d}tj|s$dSdS)a Verify UI directory has minimum required structure. Checks for: - Directory exists - Has index.html (main entry point) - Has _next directory (Next.js assets) Returns True if UI directory appears valid and servable. F index.html_nextT)rrWrexistsjoin)rZnext_dirrdrdre_validate_ui_directoryrs  rui_dirc Cstj|sdStj|d}tj|rtd|dStjtj|ds,dSz1t|D](}|rZ|j dsZtj|jd}tj|rZtd|j dWdSq2WdSt t fy}}ztd |d |WYd }~dSd }~ww) ad Detect if UI directory is already pre-restructured and ready to serve. Returns True if: 1. Marker file .litellm_ui_ready exists (created by Dockerfile), OR 2. Restructuring pattern detected (subdirectories with index.html inside) This allows skipping copy/restructure operations on read-only filesystems. Fz.litellm_ui_readyzFound UI ready marker: Trrz,Detected restructured UI via pattern: found z /index.htmlzCould not scan z for restructuring detection: N) rrWrrrrxrSris_dirr startswithPermissionErrorOSError)rZ marker_fileentryZ index_pathrerdrdre_is_ui_pre_restructureds6    r  source_path target_pathc Csz%tj|ddt|s#t|r#tj||ddtd|WdSWdSttfy?}z dt |fWYd}~Sd}~ww) z| Attempt to populate target UI directory from source. Returns: (success: bool, error_message: str) Texist_ok) dirs_exist_okzSuccessfully populated UI at )Trg)Fz(Source or target directory state invalidFN) rmakedirsrshutilcopytreerxrRrrr)r r rerdrdre_try_populate_ui_directorysrLITELLM_NON_ROOTrgtruez/var/lib/litellm/uiZLITELLM_UI_PATHzPackaged UI at z9 is invalid or incomplete. UI may not function correctly.zUsing pre-restructured UI at zUI at zT has content but is not properly restructured. Will attempt to restructure in place.zUI not found at z*. Attempting to populate from packaged UI.zFailed to populate UI at rZz!. Falling back to packaged UI at zm. For read-only deployments, pre-build UI in Dockerfile or set LITELLM_UI_PATH to a writable emptyDir volume.zUsing packaged UI directory: zSelected UI path z5 is invalid or incomplete. UI may not work correctly.rz4Cannot apply server_root_path replacements to UI at zz: path is not writable. Ensure server_root_path is '/' or pre-process UI files in Dockerfile with custom server_root_path.) z.pngz.jpgz.jpegz.gifz.icoz.woffz.woff2z.ttfz.eotrzutf-8)encodingz&/litellm/.well-known/litellm-ui-configz/.well-known/litellm-ui-configwz/_nextr directoryZ next_staticr)rhtmlZuiui_rootc Cst|D]\\}}}tj||}|dkrdn|tjd}|dvr$q|D]:}|dr1|dkr2q&tj||}tj|d}tj|d} tj |ddzt || Wq&t y`Yq&wqd S) zCEnsure each exported HTML route is available as /index.html..rgr>zlitellm-asset-prefixrz.htmlrTr N) rwalkrWrelpathrsependswithrsplitextrrr) rZ current_rootrfilesZrel_rootZ first_segmentr` file_path target_dirr rdrdre_restructure_ui_html_filesfs& r&zSkipping UI restructuring: z is already pre-restructuredzCannot restructure UI at z: path is not writable. UI may not work correctly for extensionless routes. Pre-build and restructure UI in Dockerfile for read-only deployments.zRestructured UI directory: z2Permission error while restructuring UI directory z'Error while restructuring UI directory )Z allow_originsZallow_credentialsZ allow_methodsZ allow_headersZexpose_headerscsdtjtd}tdur dnt}|ds|d}|dtjdt|dddfdd}|t _ dS)NZswaggerrz/swaggerrrcs,t|i|ddddS)Nz/swagger-ui-bundle.jsz/swagger-ui.cssz /favicon.png)Zswagger_js_urlZswagger_css_urlZswagger_favicon_urlr*)argskwargsZcustom_root_path_swagger_pathrdreswagger_monkey_patchs z.mount_swagger_ui..swagger_monkey_patch) rrWr current_dirrr!rmountr5r&r+)Zswagger_directoryZ swagger_pathr*rdr)remount_swagger_uis   r-ZROOT_REDIRECT_URLroot_redirect_urlF)include_in_schemacs ttdS)Nr)r0r.rdrdrdre root_redirects r1)rrDrsrruconfig_passthrough_endpointsz api_log.jsonrC config_agentsrNrPr)default_in_memory_ttl)Z dual_cachertallpolling_via_cache_enablednative_background_modepolling_cache_ttlhealth_check_resultsqueuezlitellm-proxy-budget)Zadminr5ui_access_moderstore_model_in_dbopen_telemetry_logger)rrr9rXcsdt}t|dkr0tdIdH|IdHr&|tdddt|dks dSdS)am Asynchronously checks if the request is disconnected at regular intervals. If the request is disconnected - cancel the litellm.router task - raises an HTTPException with status code 499 and detail "Client disconnected the request". Parameters: - request: Request: The request object to check for disconnection. Returns: - None XriNizClient disconnected the requestrdetail)timersleepZis_disconnectedcancelr)rZllm_api_call_task start_timerdrdrecheck_request_disconnection/srFcCs\ddlm}t|}|tur t|D] }t||r|SqdSt|tr,t|tr,|SdS)CResolve the actual TypedDict class from a potentially wrapped type.r)_TypedDictMetaN)Ztyping_extensionsrHrrrrrUr)typrHoriginargrdrdre_resolve_typed_dict_typeKs   rLcCspt|}g}|tur)t|D]}|dur&t|tds&dt|vr&||q|St|tr6t|tr6|gS|S)rGNNoneType)rrrrrUrrr)rIrJZtypsrKrdrdre_resolve_pydantic_typeYs    rNuse_azure_key_vaultc Cs|durdSz-ddlm}ddlm}tdd}|dur!td|}|||d}|t_t j t_ WdStyP}zt |}t d|WYd}~dSd}~ww) NFr)DefaultAzureCredential) SecretClientZAZURE_KEY_VAULT_URIzMError when loading keys from Azure Key Vault: AZURE_KEY_VAULT_URI is not set.)Z vault_url credentialztError when loading keys from Azure Key Vault: %s .Ensure you run `pip install azure-identity azure-keyvault-secrets`)Zazure.identityrPZazure.keyvault.secretsrQrgetenvr\rTZsecret_manager_clientrAZURE_KEY_VAULTZ_key_management_systemrrx exception)rOrPrQZKVUrirRclientreZ _error_strrdrdreload_from_azure_key_vaultjs,    rWcCs,tdurtjttjtdSdSrB)rNrTlogging_callback_manageradd_litellm_callbackrZ"add_litellm_async_success_callbackrdrdrdre cost_trackings rZtokenuser_id end_user_idteam_id response_costparent_otel_spanrc sgdtdtffdd }fdd}fdd} fd d } fd d } |d ur?d ur?||dId Hd urI|Id Hd urS| Id Hd ur]| Id Hd urg| Id Httjd|dd S)z` Use this to update the cache with new user spend. Put any alerting logic in here. r[r_c slt|tr|drt|d}n|}td|tj|dIdH}td||dur0dS|j}||}|j dur~|j dur~t ||j ddd ur~t ||j ddd\}}|j dtd }t|jpgd ||j||j||tjd } ttjd | d|durt|dddur|jpd} | ||_|durt|dddur|jpd} | ||_||_||fdS)Nzsk-r[z"_update_key_cache: hashed_token=%skeyz&_update_key_cache: existing_spend_obj=FZ soft_budget)Z current_spendZsoft_budget_limitTinfrg)r[spend key_aliasrr\projected_spendprojected_exceeded_dateZ event_groupZprojected_limit_exceeded)rUZ user_info team_spendrteam_member_spend)rrrrrxrSrrasync_get_cachereZsoft_budget_cooldownZlitellm_budget_tablerrrfloatr(r[rfr\r7ZKEYrrrqZ budget_alertsrrirjr) r[r_Z hashed_tokenexisting_spend_objexisting_spend new_spendrgrhZ soft_limitZ call_infoZexisting_team_spendZexisting_team_member_spend)values_to_update_in_cacherdre_update_key_cachesv           z'update_cache.._update_key_cachec sVg}z|D]Q}|durqtj|dIdH}|durWdStd|dt|tr4|d}n|j}|}t|trL||d<||fq||_||fqtjd t dIdH}|durlWdSdur|dur|}d t |fWdSWdSWdSt y}zt dt |tWYd}~dSd}~ww)Nrbz!_update_user_db: existing spend: ; response_cost: re{}:spendzSpend tracking - failed to update user spend in cache. Budget enforcement may use stale spend values. user_id=%s, response_cost=%s - %s %s)rrrkrxrSrrrerrrqrzr\rcr traceback format_exc)Zuser_ids_idrmrnroZglobal_proxy_spend incrementre)r_r\rprdre_update_user_cachesT      z(update_cache.._update_user_cachec s dus dur dSd}zUtj|dIdH}|dur!WdStd|d|dur3d}n t|tr=|d}n|j}|}t|trW||d<||fWdS||_|| fWdSt y}zt dt |t WYd}~dSd}~ww)Nzend_user_id:{}rbz%_update_end_user_db: existing spend: rrrrezSpend tracking - failed to update end user spend in cache. Budget enforcement may use stale spend values. end_user_id=%s, response_cost=%s - %s %s)rqrrrkrxrSrrrerrr\rcrrtrurvrmrnrore)r]r_rprdre_update_end_user_cache-s@    z,update_cache.._update_end_user_cachec sdus dur dSd}zYtj|dIdH}|dur!WdStd|d|dur3d}n t|tr=|d}n|j}|durFd}|}t|tr]||d<||fWdS||_||fWdSt y}zt dt |t WYd}~dSd}~ww)Nz team_id:{}rbz!_update_team_db: existing spend: rrgrezSpend tracking - failed to update team spend in cache. Budget enforcement may use stale spend values. team_id=%s, response_cost=%s - %s %s)rqrrrkrxrSrrrerr\rcrrtrury)r_r^rprdre_update_team_cacheYsF    z(update_cache.._update_team_cachec s$dus dur dSzfD]`}|rt|tsqd|}tj|dIdH}|dur+qtd|d|dt|trG|ddpEd}nt|ddpNd}|}t|trd||d< ||fq||_  ||fqWdSt y}zt d t|t WYd}~dSd}~ww) z: Update the tag cache with the new spend. Nztag:rbz*_update_tag_cache: existing spend for tag=rZrrrerzSpend tracking - failed to update tag spend in cache. Budget enforcement may use stale spend values. tags=%s, response_cost=%s - %s %s)rrrrrkrxrSrrrrrer\rcrtru)Ztag_name cache_keyZexisting_tag_objrnrore)r_rrprdre_update_tag_cachesH     z'update_cache.._update_tag_cacheN)r[r_rj)Z cache_listttlZlitellm_parent_otel_span)rrlrrrrZasync_set_cache_pipeline) r[r\r]r^r_r`rrqrxrzr{r}rd)r]r_rr^r\rpre update_caches0S6,03    rc Csz'ddg}ttjd}tj|||dWdWdS1s wYWdStyC}ztd|dWYd}~dSd}~ww)NollamaZserver)stdoutstderrzd LiteLLM Warning: proxy started with `ollama` model `ollama serve` failed with Exceptionz). Ensure you run `ollama serve` )openrdevnull subprocessPopenr\rxrS)commandrrerdrdrerun_ollama_serves&rcCsRzddl}||jj}tjdkrt|dWSt|dWSty(YdSw)zj Get process RSS memory in MB. On Linux, ru_maxrss is in KB. On macOS, ru_maxrss is in bytes. rNdarwinii)resourceZ getrusageZ RUSAGE_SELF ru_maxrsssysplatformrlr\)rrrdrdre_get_process_rss_mbs  rcCst}|dur dS|dS)Nunknown.2f)r)Zrss_mbrdrdre_rss_mb_for_logsr model_listdetailsmax_concurrencyinstrumentation_contextc sbz t||||dIdHWSty0}zdt|vrt|||dIdHWYd}~Sd}~ww)N)rrrrrrrr)rrr)rrrrrerdrdre-_run_direct_health_check_with_instrumentations$ rhealthy_endpointsunhealthy_endpointsc CsX|durdSddl}ddlm}|dur|jnd}|}t|||||||ddS)zKFire-and-forget: persist health check results to DB if prisma is available.Nr)$_save_background_health_checks_to_dbZbackground_health_check) checked_by)rB0litellm.proxy.health_endpoints._health_endpointsrZpod_idrr) rNshared_health_managerrrrZ time_modulerrrErdrdre)_schedule_background_health_check_db_saves& rc sftdustttrtdkrdStrtdtttdatdtttt t t d}trDt durDddlm}|t ttd}td td 7ad t}t}ttpXg}t|}d d |D}t|}|}tttr|tdkr||dkr|t|t}td |||tt||dut t dd|d}t durt nd} |durz|j|| tdIdH\} } Wn2ty} ztdt| t|t t|IdH\} } WYd} ~ nd} ~ wwt|t t|IdH\} } | t d<| t d<t| t d<t| t d<t|d} td||t| t| | tt t | tdkr"td|| tt!t"||| | t#$tIdHqE)z Periodically run health checks in the background on the endpoints. Update health_check_results, based on this. Uses shared health check state when Redis is available to coordinate across pods. Nrzxbackground_health_check_loop_overlap_detected existing_loop_active=true interval_seconds=%s max_concurrency=%s shared=%sTzzbackground_health_check_loop_started interval_seconds=%s max_concurrency=%s shared=%s details=%s thread_count=%d rss_mb=%s)SharedHealthCheckManager) redis_cacheZhealth_check_ttlZlock_ttlz'Initialized shared health check managerrizbg-cSs$g|]}|didds|qS) model_infoZdisable_background_health_checkFrrmrdrdrerqs z0_run_background_health_check..zbackground_health_check_cycle_start cycle_id=%s model_count_total=%d model_count_enabled=%d interval_seconds=%s max_concurrency=%s expected_peak_in_flight=%d shared=%s thread_count=%d rss_mb=%sZproxy_background_loop)enabledsourcecycle_idrzEError in shared health check, falling back to direct health check: %srrZ healthy_countZunhealthy_countizbackground_health_check_cycle_complete cycle_id=%s model_count_enabled=%d healthy_count=%d unhealthy_count=%d duration_ms=%.2f interval_seconds=%s thread_count=%d rss_mb=%szibackground_health_check_cycle_duration_exceeded_interval cycle_id=%s duration_ms=%.2f interval_seconds=%s)%rLrr#background_health_check_loop_activerxrcrMrKrRhealth_check_details threading active_countrrtZZ$ dd?e ede efd@dAZ%dde&fdCdDZ'dEe(defdFdGZ)dEe(defdHdIZ*dJe(de(fdKdLZ+dJe e,dMe-fdNdOZ.dPedQee/dRdSe(ddfdTdUZ0dVeddfdWdXZ1 ddYedZe edefd[d\Z2 BddYed]edefd^d_Z3d`edefdadbZ4e5dce6de efdddeZ7 ddfe dgd6e e8dMe dhde efdidjZ9dVed8e e d6e e8ddfdkdlZ:dVedd6e8dedze efd{d|Z?d}e@eeAfdefd~dZBd6e8de(fddZCd6e8dMe-fddZDd6e8fddZEd6e8fddZFd6e8fddZGd6e8fddZHd6e8fddZId6e8fddZJddZKd6e8fddZLd6e8fddZMd6e8fddZNd6e8fddZOd6e8fddZPd6e8fddZQddZRd6e8fddZSd6e8fddZTddZUde@eeVfdefddZWdeefddZXd6e8fddZYdS) ProxyConfigz} Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic. rNcCsi|_d|_d|_dSrB)config_last_semantic_filter_config_last_hashicorp_vault_configselfrdrdre__init__s zProxyConfig.__init__rmcCs8tj|sdStj|\}}|dkp|dkS)NFz.yamlz.yml)rrWrr"lower)rrmrZfile_extensionrdrdrers zProxyConfig.is_yamlr$c Csrz t|d}t|p iWdWS1swYWdSty8}z td|dt|d}~ww)z, Load and parse a YAML file rNzError loading yaml file rZ)ryaml safe_loadr\r)rr$rbrerdrdre_load_yaml_files  (zProxyConfig._load_yaml_filecs|pt}|dur |atj|r.t|d }t|}Wdn1s(wYn|dur9td|giiid}|durHtd|j|tj tj |pTdd}|S)z Given a config file path, load the config from the file. Args: config_file_path (str): path to the config file Returns: dict: config NrzConfig file not found: )rrurouter_settingsrvzConfig cannot be None or Empty.rg)rbase_dir) rDrrWrrrrr\_process_includesdirnameabspath)rrmr$ config_filerrdrdre_get_config_from_files,  z!ProxyConfig._get_config_from_filerrcCsd|vr|St|dtstd|dD]9}tj||}tj|s+td|||}| D]\}}t|trI||vrI|| |q4|||<q4q|d=|S)au Process includes by appending their contents to the main config Handles nested config.yamls with `include` section Example config: This will get the contents from files in `include` and append it ```yaml include: - model_config.yaml litellm_settings: callbacks: ["prometheus"] ``` includez&'include' must be a list of file pathszIncluded file not found: ) rrrrrWrrrritemsextend)rrrZ include_filer$Zincluded_configrcrrdrdrer' s     zProxyConfig._process_includes new_configcs tdur?tdddustr?|}d|vr-|dr-|j|ddd}|j|d|d<|ddtj|dd IdHdSt t d }t j ||dd WddS1sYwYdS) Nr=FTenvironment_variables)rreturn_original_valuerrr)data table_namer)Zdefault_flow_style) rNrurr=r!_decrypt_and_set_db_env_variables_encrypt_env_variablespopZ insert_datarrDrdump)rrZconfig_to_savedecrypted_env_varsrrdrdre save_configN s, "zProxyConfig.save_configrdepth max_depthcCs||krtd|d|S|D]@\}}t|tr)|j||d|d||<qt|trC|D]}t|trA|j||d|d}q0qt|trS|drSt |||<q|S)a Check for os.environ/ variables in the config and replace them with the actual values. Includes a depth limit to prevent infinite recursion. Args: config (dict): The configuration dictionary to process. depth (int): Current recursion depth. max_depth (int): Maximum allowed recursion depth. Returns: dict: Processed configuration dictionary. zMaximum recursion depth (z") reached while processing config.ri)rrr os.environ/) rxrcrrr_check_for_os_environ_varsrrrr)rrrrrcritemrdrdrerx s,       z&ProxyConfig._check_for_os_environ_varsr^all_teams_configcCsti}|D]}d|vrtdt|||dkr|}nq|D]\}}t|tr7|dr7t|||<q#|S)Nr^zteam_id missing from team: r)r\SENSITIVE_DATA_MASKERZ mask_dictrrrrr)rr^r team_configteamkvrdrdre_get_team_config s  zProxyConfig._get_team_configcCs>|}|di}|dd}|duriS|j||d}|S)z^ - for a given team id - return the relevant completion() call params rvdefault_team_settingsN)r^r)rrr)rr^rrvrrrdrdreload_team_config s  zProxyConfig.load_team_config cache_paramscCspddlm}d|vr|dt_d|vr|dt_|di|t_tjdur4ttjjttfr6tjjadSdSdS)NrCacher4default_redis_ttlrd) rTrr4rrUrr{r|rt)rrrrdrdre _init_cache s      zProxyConfig._init_cachecCsDddl}tdur|jdurtjdur dt_tddSdSdSdS)z Enable caching on the router by setting cache_responses=True. This ensures caching works without needing caching=True in request body. Router passes caching=self.cache_responses to litellm.completion() rNTz8Set router.cache_responses=True after initializing cache)rTrsrUcache_responsesrxrS)rrTrdrdreswitch_on_llm_response_caching s  z*ProxyConfig.switch_on_llm_response_cachingcstjddur?tjd}tjd}tjd}td|||dkr0t||dIdH}nt||d}|dur>tdn |j|d IdH}t dur[t d ur[|j |t t d IdH}t |}|d d|j|d }|j|d |S)a  Load config file Supports reading from: - .yaml file paths - LiteLLM connected DB - GCS - S3 Args: config_file_path (str): path to the config file Returns: dict: config roNZ LITELLM_CONFIG_BUCKET_OBJECT_KEYZLITELLM_CONFIG_BUCKET_TYPEzbucket_name: %s, object_key: %sZgcs) bucket_name object_keyz(Unable to load config from given source.rlT)rrNr=rr)rrrrxrSrrr\rrNr=_update_config_from_dbrrrrupdate_config_state)rrmrrZ bucket_typerZ printed_yamlrdrdre get_config s<       zProxyConfig.get_configcCs ||_dSrBr)rrrdrdrer" s zProxyConfig.update_config_statec CsLzt|jWSty%}ztd|j|iWYd}~Sd}~ww)z Returns a deep copy of the config, Do this, to avoid mutating the config state outside of allowed methods z^ProxyConfig:get_config_state(): Error returning copy of config state. self.config={} Error: {}N)rrrr\rxrSrq)rrerdrdrer% szProxyConfig.get_config_statecCs$|d}g}|rdd|D}|S)z< Load the credential list from the database credential_listcSg|] }tdi|qSrd)rrcredrdrdrer< z4ProxyConfig.load_credential_list..r)rrcredential_list_dictrrdrdreload_credential_list5 s z ProxyConfig.load_credential_listc Cs@|dd}|s|di}|r|dd}|sdSg}td|D]w}|dd}|didd}td |d |d |di}|rl|D]\} } t| trg| d rg| d d} t| } | || <qK||d<ztdi|} | | Wq"t y} zt d |dt| WYd} ~ q"d} ~ ww|r|SdS)a/ Parse and validate search tools from config. Loads environment variables and casts to SearchToolTypedDict. Args: config: Config dictionary containing search_tools Returns: List of validated SearchToolTypedDict or None if not configured search_toolsNruz6LiteLLM: Proxy initialized with Search Tools:search_tool_namerglitellm_paramssearch_provider  z (z)rzError parsing search tool rZrd) rrprrrrrrrrr\rxr)rrZsearch_tools_rawruZsearch_tools_parsedZ search_toolrrrrr_vZsearch_tool_typedrerdrdreparse_search_tools? sJ         zProxyConfig.parse_search_toolscCs|dd}|r@|D]$\}}t|tr)|dr)t|d}|dur(|tj|<q t|tj|<q d|vr@tddt _ t a dS)Nrr)Z secret_nameZLITELLM_LICENSE) rrrrrrrrrSr license_strrr9)rrrrcrZresolved_secret_stringrdrdre_load_environment_variablesx s   z'ProxyConfig._load_environment_variablesrc? s, |jdIdH}|j|d|di}|r|t_|dd}|dur'i}|rQd}d}|D]\}} |dkr| d urt|d d d lm} i} d |vr[|d } | | | dd} t d| | dkso| dkrt | d krtdd}tdd}d}| | ||dtdddurtdd}| d|it d||| t d|||t d|||t d|||| D]\}} t| tr| drt| | |<q|j| dtjdurt |d|q2|dkr| durq2|dkrt| t|d }|t_q2|d!kr"d d"lm}|| t |d#|q2|d$kr|2qt|d?tj-d@|q2|dAkrYgt_3| D]}d:|vrCtj.4t%|d;q1tj.4|q1t|dBtj3d@|q2|d kr_q2|dCkr| dDi}|dEda5|dFga6|dGdHa7t |dIt5dJt6dKt7|q2|dLkrt8| D]4\}}z t9di|Wqt:yt|t;rt:dM|dN| t:dM|dNt<|wt |dO|dP| |t=t|| q2|dQkr| durt| t;r| D]\}}t|tr |dr t|| |<qt>di| t_?q2t:dR| |dSkr;| d ur;d t_@tAt |dT|q2t |dO|dP| |t=t|| q2|dUi}|dur^i}|r |dVd} | durttBdi| t_C|dWd}!|jD|!dX|dYd}"tE|"dZ|d[d}#tF|#d\|jG|d]|d^d}$|$r|$drt d_t|$}$t d`|$|datdbdaHtHrtHdrttHaHtHdurttHtrtItHaJnt Kdc|ddd}%|%durtLjMt'|%dde|dfdaNtNdurdaNtN|df<|dgd}&|&durt%|&d1aO|dhd}'|'dur-t%|'d1aP|did}(|(dur>t%|(d1aQ|djd})|)durOt%|)d1aRtSdur\tST|IdH|dkddurs|dkaUtV|dkdlIdH|dmdnaW|dod}*|*durtdurtXdp|dqtYaY|drtZaZ|dst[a[|dtt\a\|dut]a]|dvda^|dwda_|dxt`aa|dydab|dzd act d{t^t_tatbtc|d|d}+|+durd}d8|+D|d|<|d~durtd urtXdtdjejfd|vr |dtg_htgiadtjdui},|dd}-|-ro|-|,d<td|-D]J}.|.dD]\}/}0t|0trD|0drDt|0|.d|/<q,td|.ddd|.dd}1|.ddd}2d|1vrm|2durmtjq$d}3|dd}4|4r|4dD]"\}/}0t|0tr|0dr|0kdd}tlm|}0|0|4d|/<qtndi|4}3|o|}5|dd}6tp|6d|dd}7tq|7d|dd}8trjs|8d|dd}9|9rt|9t;rddhfdd8tjtuD}:|9D]&\}/}0|/|:vr|0|,|/<q|/dvrtXd|/dt vd|/dqtjtdi|,|3|5twd dd d}txdur?|jjydur?|jztxdd};|durL|dd};|;rVt{|;|d|j||t}|dIdHd}<|durn|dd}<|<r}d dl~m}=|=|<d|j|d}>|>t_|j|dIdH|||fS)z< Load config values into proxy global state rlNrcallback_settingsrvzzrUTz Setting Cache on ProxyrrrrUZrediszpassed cache type=%szredis-semanticZ REDIS_HOSTZ REDIS_PORT)rUhostportZREDIS_PASSWORDpasswordz%sCache Type:%s %sz%sCache Host:%s %sz%sCache Port:%s %sz%sCache Password:%s %sr)rzSet Cache on LiteLLM ProxyF guardrails)Zguardrails_configr9rmrvZglobal_prompt_directory)set_global_prompt_directoryz,Set Global Prompt Directory on LiteLLM ProxyZglobal_bitbucket_config)set_global_bitbucket_configz,Set Global BitBucket Config on LiteLLM ProxyZglobal_gitlab_config)set_global_gitlab_configz)Set Global Gitlab Config on LiteLLM Proxypriority_reservation_settings)PriorityReservationSettings callbacks)rr9rmrvmodel_group_settings)ModelGroupSettingspost_call_rulesrrmzlitellm.post_call_rules: max_internal_user_budget default_max_internal_user_budgetcustom_provider_map)custom_llm_setupcs&g|]}|dt|dddqS)providercustom_handlerr)rrrrrrlrdrerH sz+ProxyConfig.load_config..rYrrZ prometheusPrometheusLoggerzmounting metrics endpointz! Initialized Success Callbacks - rlfailure_callbackz! Initialized Failure Callbacks - rbackground_modeZpolling_via_cacher7r~r8z( Initialized polling via cache: enabled=z, native_background_mode=z, ttl=rz4team_id missing from default_team_settings at index=z passed in value=z setting litellm.=upperbound_key_generate_paramsz=Invalid value set for upperbound_key_generate_params - value= json_logsz Enabled JSON logging via configrukey_management_settingskey_management_system)rrmuse_google_kmsr rOrOrurpzGOING INTO LITELLM.GET_SECRET!zRETRIEVED DB URL: %srCrjzLITELLM_MASTER_KEY is not set! All requests will be treated as INTERNAL_USER with no admin access. Set LITELLM_MASTER_KEY for production use.user_api_key_cache_ttl)r4rr= custom_authcustom_key_generate custom_ssocustom_ui_sso_sign_in_handlerpass_through_endpointsr)r<r5 allowed_ipsz\allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment.r{r|proxy_batch_polling_intervalr}disable_spend_logsZbackground_health_checksrKrLrMrzebackground_health_check_config enabled=%s shared=%s interval_seconds=%s max_concurrency=%s details=%sZrole_permissionscSrr)r?)rZrole_permissionrdrdrer^ s Zenforced_paramszTrying to use `enforced_params`Zlitellm_licenserrz<LiteLLM: Proxy initialized with Config, Set models:rr model_namergmodelapi_baserassistant_settingsZfinetune_settingsZfiles_settingsdefault_vertex_configrrcsg|]}|vr|qSrdrd)rx) exclude_argsrdrer s>rLrM'zQ' is NOT a valid router_settings parameter. Please move it to 'general_settings'.zKey 'zC' is not a valid argument for Router.__init__(). Ignoring this key.Zasync_only_mode)assistants_configrrouter_general_settingsignore_invalid_deployments)rU)Zall_guardrailsrmrs)rrNrsprompts) init_prompts)Z all_promptsrmrd)rrrrTrrrplitellm.caching.cachingrupdaterxrSrrrrrrrrUrr9guardrail_name_config_mapZlitellm.integrations.dotpromptrrRZlitellm.integrations.bitbucketrZlitellm.integrations.gitlabrlitellm.types.utilsr rrlitellm.types.routerr r rr rlrrrZrrrYrXadd_litellm_success_callbacklitellm.integrations.prometheusrZ_mount_metrics_endpointradd_litellm_failure_callbackr6r7r9 enumeraterBr\rrUsetattrr rrZ _turn_on_jsonr_key_management_settingsinitialize_secret_managerrrW_load_alerting_settingsrCrlitellm_master_key_hashcriticalrrZupdate_cache_ttlr=rFrGrHrIr8Zload_enterprise_configr2rr<rr{r|r,r}r-rJrKrrLrMrr)Znot_premium_userrrrrrrrrSrrrrrZset_default_vertex_configrvZget_valid_argsrcr rtrZ_update_redis_cacher_init_policy_enginerNZ"litellm.proxy.prompts.init_promptsr;rr_init_non_llm_configsget_model_list)?rrrmrrrvZblue_color_codeZreset_color_codercrrrZcache_params_in_configZ cache_typeZ cache_hostZ cache_portZcache_passwordr>rrrr r rcallbackrridxZ team_settingZ_krrurrr rOrpr$r%r&r'r(r+Zrbac_role_permissionsZ router_paramsrr/rrlitellm_model_nameZlitellm_model_api_baser7r1rZfinetuning_configZ files_configr2rZavailable_argsZ guardrails_v2r:r;rrd)rmr4rer s                                                                                                                       zProxyConfig.load_configc s|dd}|rt||dd}|rt||dd}|r>ddlm}|di}|dd}|||IdH|d d}|r[dd lm } t j durU| t _ t j |dS) zO Initialize non-LLM configs eg. MCP tools, vector stores, etc. Z mcp_toolsNZ agent_listZ mcp_serversrglobal_mcp_server_managerrv mcp_aliasesvector_store_registryVectorStoreRegistry) rrZload_tools_from_configrZload_agents_from_config9litellm.proxy._experimental.mcp_server.mcp_server_managerrRZload_servers_from_config+litellm.vector_stores.vector_store_registryrVrTrTZload_vector_stores_from_config) rrZmcp_tools_config agent_configZmcp_servers_configrRrvrSZvector_store_registry_configrVrdrdrerL s0           z!ProxyConfig._init_non_llm_configsrNrrsrvcsddlm}ddlm}|durtddS|dd}|s'tddS|dd}td t|d |||||dud d IdHdS) a Initialize the policy engine from config. Args: config: The proxy configuration dictionary prisma_client: Optional Prisma client for DB validation llm_router: Optional LLM router for model validation r) init_policies)PolicyValidatorNz'Policy engine: config is None, skippingpoliciesz.Policy engine: no policies in config, skippingZpolicy_attachmentszPolicy engine: found z policies in configT)policies_configpolicy_attachments_configrNZ validate_dbZ fail_on_error) Z)litellm.proxy.policy_engine.init_policiesrZZ,litellm.proxy.policy_engine.policy_validatorr[rxrSrrRr)rrrNrsrZr[r]r^rdrdrerK s*      zProxyConfig._init_policy_engineruc Cs|dd}td||durdS|dd}td|tj||dd|dd|dd|d d|d dtd |D]$}|d krKqD|tjvrht|ddd |d did }|durhtj |qDdS)z. Initialize alerting settings alertingNz_alerting_callbacks: z=_load_alerting_settings: Calling update_values with alerting=alerting_thresholdr? alert_typesalert_to_webhook_url alerting_argsalert_type_config)r_r`rarbrcrdrslack)Zlogging_integrationZinternal_usage_cachersZcustom_logger_init_args) rrxrSrq update_valuesrtrT)_known_custom_logger_compatible_callbacksr%rXrY)rruZ_alerting_callbacksZ_alerting_valueZ_alertZ_loggerrdrdrerHE s@           z#ProxyConfig._load_alerting_settingsrcCs|dur|tjjkrtdddS|tjjkrtdddS|tjjkr4ddlm}|j dt j ddS|tj jkrAt dddS|tjjkrRdd lm}|dS|tjjkrcdd lm}|dS|tjjkrtdd lm}|dS|tjjkrdd lm}||d dStddS)z_ Initialize the relevant secret manager if `key_management_system` is provided NTr"r!r)AWSSecretsManagerV2)Zuse_aws_secret_managerr)Z use_aws_kms)GoogleSecretManager)HashicorpSecretManager)CyberArkSecretManager)load_custom_secret_managerrlz&Invalid Key Management System selected)rrTrrWZ GOOGLE_KMSrZAWS_SECRET_MANAGERZ-litellm.secret_managers.aws_secret_manager_v2rhZload_aws_secret_managerrTrFZAWS_KMSrZGOOGLE_SECRET_MANAGERZ-litellm.secret_managers.google_secret_managerriZHASHICORP_VAULTZ0litellm.secret_managers.hashicorp_secret_managerrjZCYBERARKZ/litellm.secret_managers.cyberark_secret_managerrkZCUSTOMZ4litellm.secret_managers.custom_secret_manager_loaderrlr)rrrmrhrirjrkrlrdrdrerGq s:               z%ProxyConfig.initialize_secret_managerFcCst|dd}|dur||jd<d|jd<tdur flag for differentiating model stored in db vs. config -> used on UI Return model info w/ id model_idNrbTdb_model created_at updated_at created_by updated_byF)rbrnrd)rrr9rrrmRouterModelInfo)rr/rnrv _model_infordrdreget_model_info_with_id s$      z"ProxyConfig.get_model_info_with_id db_modelsc spg}tdus t|dkrdS|D]}|j|d}|jdur$||jqz |jtdIdH}WntyK}zt dt |WYd}~dSd}~ww| dd}|r|D]B}|d D]\} } t | t rt| drtt| |d| <q^| d i d d} | durtj|d |dd } nt | } || qVt} d} | D]} | |vrtj| d }|dur| d7} q| S)a5 (Helper function of add deployment) -> combined to reduce prisma db calls - Create all up list of model id's (db + config) - Compare all up list to router model id's - Remove any that are missing Return: - int - returns number of deleted deployments Nrr/rlzlFailed to load config in _delete_deployment: %s. Skipping deployment cleanup to avoid removing valid models.rrrrrbr.) model_grouprrbri)rsrrurbrrrDr\rxrcrrrrrrZ_generate_model_id get_model_idsZdelete_deployment)rrvZcombined_id_listrrrrerr/rrrmZrouter_model_idsZdeleted_deploymentsZ is_deletedrdrdre_delete_deployment sV       zProxyConfig._delete_deploymentc Csddl}tdur dSd}|D]O}|j}t|tr9|D]\}}t|tr0t||dd}|||<qtd i|}n t d|q|j |dd} tj t |j|| dd} | dur]|d 7}q|S) z Iterate through db models for any not in router - add them. Return - number of deployments added rNTrrcrHInvalid model added to proxy db. Invalid litellm params. litellm_params=)r/rnr.rr) deploymentrird)base64rsrrrrrrrrxrruZupsert_deploymentrr.) rrvrZ added_modelsr_litellm_paramsrr_valuertaddedrdrdre_add_deploymentsB  zProxyConfig._add_deployment new_modelsc Csg}|D]M}|j}t|tr|}t|tr3|D]\}}t||dd}|||<qtdi|}n t d|q|j |d}| t |j ||djddq|S)NTr|r}rwr~ exclude_nonerd)rrr model_dumprrrrrxrrurrr.to_json) rr _model_listrrrrdecrypted_valuertrdrdredecrypt_model_list_from_db?s6    z&ProxyConfig.decrypt_model_list_from_dbrqc si}d}ztIdH}||}Wnty-}ztdt|WYd}~nd}~wwzlt|tr6|ng}t durut durut dt ||j |d}t |dksX|rtt d|tj|tdd|dda t d t n$t dt ||durt dur|t _|j|d IdH|j|d Wnty}ztd t|WYd}~nd}~wwt durt a|||j|t td IdH|j|t|d dS)NziFailed to load config in _update_llm_router: %s. Proceeding with model loading using cached/empty config.zlen new_models: )rrz _model_list: Tr6)rr8rr9zupdated llm_router: )rvz+Error adding/deleting model to llm_router: ) config_datarsrN)rrurq)rrrr\rxrcrrrrsrCrSrrrTrvr rr{rrUrMr_add_callbacks_from_db_config#_add_router_settings_from_db_configrN$_add_general_settings_from_db_configru)rrrqrrreZ models_listrrdrdre_update_llm_router^sn     zProxyConfig._update_llm_routerrN event_types)successfailureexisting_callbackscCsr|tjvr|D]}t||qdS||vr7|dgkr"tj|dS|dgkr/tj|dStj|dSdS)aQ Helper method to add a single callback to litellm for specified event types. Args: callback: The callback name to add event_types: List of event types (e.g., ["success"], ["failure"], or ["success", "failure"]) existing_callbacks: The existing callback list to check against rrN)rTrgrrXrArCrY)rrNrrZ event_typerdrdre4_add_callback_from_db_to_in_memory_litellm_callbackss    z@ProxyConfig._add_callback_from_db_to_in_memory_litellm_callbacksrc Cs|dipi}|dd}|dd}|dd}|dur2t|tr2|D] }|j|dgtjdq%|durJt|trJ|D] }|j|dgtjdq=|durct|tre|D]}|j|ddgtjdqUdSdSdS) z: Adds callbacks from DB config to litellm rvrYNrr r)rNrrr)rrrrrTrYrr ) rrrvsuccess_callbacksZfailure_callbacksr rYrrNrdrdrers6   z)ProxyConfig._add_callbacks_from_db_configrnew_encryption_keycCs.i}|D]\}}t||d}|||<q|S)zR Encrypts a dictionary of environment variables and returns them. )rr)rr)rrrZencrypted_env_varsrrencrypted_valuerdrdrers z"ProxyConfig._encrypt_env_variablesrc Cs|i}|D]5\}}zt|||d}|dur|tj|<|||<Wqty;}ztd|t|WYd}~qd}~ww|S)a! Decrypts a dictionary of environment variables and then sets them in the environment Args: environment_variables: dict - dictionary of environment variables to decrypt and set eg. `{"LANGFUSE_PUBLIC_KEY": "kFiKa1VZukMmD8RB6WXB9F......."}` r|Nz#Error setting env variable: %s - %s)rrrrr\rxrr)rrrrrrrrerdrdrers"   z-ProxyConfig._decrypt_and_set_db_env_variablesvariables_dictcCs0i}|D]\}}t||dd}|||<q|S)zF Decrypts a dictionary of variables and returns them. Tr|)rr)rrZdecrypted_variablesrrrrdrdre_decrypt_db_variables s z!ProxyConfig._decrypt_db_variablesrc Cs|durdSd}t|tr|}n0t|tr@ddl}zt|}Wntj|jfy?z||}Wn |jy<YnwYnwt|trI|rI|SdS)z Parse a router_settings value that may be a dict or a JSON/YAML string. Returns a non-empty dict if valid, otherwise None. Nr) rrrrrrZ YAMLErrorJSONDecodeErrorr)rparsedrrdrdre_parse_router_settings_values(  z(ProxyConfig._parse_router_settings_valueuser_api_key_dictrErcs|dur|t|dd}|dur|S|durG|jdurGzt|j|t|dIdH}|t|dd}|dur:|WSWdStyFYdSwdS)u Get router_settings in priority order: Key > Team Uses the already-cached key object and the cached team lookup (get_team_object) to avoid direct DB queries on the hot path. Global router_settings are NOT looked up here — they are already applied to the Router object at config-load / DB-sync time. Returns: dict: router_settings, or None if no settings found Nr)r^rNrrrq)rrr^rrrr\)rrrNrqZ key_settingsZteam_objZ team_settingsrdrdre!_get_hierarchical_router_settings1s6    z-ProxyConfig._get_hierarchical_router_settingscs|durc|dure|jjjddidIdH}|di}i}|dur>t|tr>|dur>t|jtr>ddlm}|||j}n|durJt|trJ|}n |durWt|jtrW|j}|rg|j di|dSdSdSdS)z Adds router settings from DB config to litellm proxy 1. Get router settings from DB 2. Get router settings from config 3. Combine both 4. Update router settings N param_namerwherer)_update_dictionaryrd) r~litellm_config find_firstrrr param_valuerZrZupdate_settings)rrrsrNZdb_router_settingsZconfig_router_settingsZcombined_router_settingsrrdrdrercs:     z/ProxyConfig._add_router_settings_from_db_configcs|di}|durd|vrdursdddurstdtrs|dddurst|dtrstd}t|d}t||}tdfdd|dD}tddd|dd||d<|jdd n(duri|dd<|jdd nttr|dd<|jdd |durd |vr|d d <d |_ |j jd t d |durd |vr|d d <|j jd t d dSdSdS)z Adds general settings from DB config to litellm proxy Args: config_data: dict general_settings: dict - global general_settings currently in use proxy_logging_obj: ProxyLogging ruNr_csg|] }|dvr|qSr_rdrr#rdrers  zDProxyConfig._add_general_settings_from_db_config..zMerging alerting values: YAML=z, DB=z , Merged=rra)rarsrb)rbrs) rrrsetunionrxrSrfrraslack_alerting_instancers)rrrurq_general_settingsZ_yaml_alertingZ _db_alertingZ_merged_alertingrdr#rersd           z0ProxyConfig._add_general_settings_from_db_configc sZtdurdSz tdtdWn tyYnwtd}|durddlm}|}td}|rlddl m }z| |}tj |j |tgdd td td |WdStyktd |YdSwdd lm}tdd}z#||} tj |j d| tddtgdd tdtd|WdStytdYdSwdS)z Reschedule the spend log cleanup job based on current general_settings. This is called when maximum_spend_logs_retention_period is updated dynamically. If the retention period is None, the job will be removed. Nspend_log_cleanup_jobz&Removed existing spend log cleanup job#maximum_spend_logs_retention_periodrrmaximum_spend_logs_cleanup_cron CronTriggerTr'rbreplace_existingmisfire_grace_timez)Spend log cleanup rescheduled with cron: /Invalid maximum_spend_logs_cleanup_cron value: )r%maximum_spend_logs_retention_interval1dintervalrjsecondsr'rbrrz-Spend log cleanup rescheduled with interval: 3Invalid maximum_spend_logs_retention_interval value) schedulerZ remove_jobrxrRr\rur7litellm.proxy.db.db_transaction_queue.spend_log_cleanuprapscheduler.triggers.cronr from_crontabadd_jobcleanup_old_spend_logsrNrrrZ*litellm.litellm_core_utils.duration_parserrrmrandint) rZretention_periodrspend_log_cleanup cleanup_cronr cron_triggerrretention_intervalinterval_secondsrdrdre!_reschedule_spend_log_cleanup_jobst             z-ProxyConfig._reschedule_spend_log_cleanup_jobdb_general_settingscs|durdSt|}d|vr|dtd<d|vr|dtd<d|vr2|dtd<tjjtddd|vrF|dtd<ttddIdHd|vrP|dtd<d |vr|d }|duradtd <nt|trk|td <nt|try| d ktd <nt|td <d |vr|d }|durnt|tr|a nt|tr| d ka nt|a t td <d |vrt d }|d }|td <||kr| IdHdSdSdS) z; Pull from DB, read general settings value Nmax_parallel_requestsglobal_max_parallel_requestsrc)rcr)r*r<store_prompts_in_spend_logsrr=r) rrurqrrfrrboolrrr=rr)rrrr old_value new_valuerdrdre_update_general_settings#sl            z$ProxyConfig._update_general_settingscurrent_configrrurrvrdb_param_valuec Csdtdtddfdd}|dkr=|j|dd }i}|D]\}}|||<|} ||| <|tj| <q|di||S|d krYt|trY|D]\}}|t vrXt t ||qJ||vrc|||<|St||trxt|trx|||||S|||<|S) am Updates the config fields with the new values from the DB Args: current_config (dict): Current configuration dictionary to update param_name (Literal): Name of the parameter to update db_param_value (Any): New value from the database Returns: dict: Updated configuration dictionary dstsrcrNcSs||fg}|rF|\}}|D]0\}}|durqt|tr&t|dkr&qt|tr=t||tr=||||fq|||<q|sdSdS)z Deep-merge src into dst, skipping None values and empty lists from src. On conflicts, src (DB) wins, but empty lists are treated as "no value" and don't overwrite. Nr)rrrrrrrr)rrstackdsrrrdrdre_deep_merge_dictss    z._deep_merge_dictsrT)rrv) rrrupperrr setdefaultr=rr"rErT) rrrrrrZmerged_env_varsrcrZ upper_keyrdrdre_update_config_fieldsqs<   z!ProxyConfig._update_config_fieldsr=c s|dur td|Sg}gd}|D]}|jd|dd}||qtj|IdH}|D].}|dur5q.t|dd} t|dd} td| d | | dur\| dur\|j|| | d }q.|S) NTz4'store_model_in_db' is not True, skipping db updatesrrr)rcrrrz param_name=z, param_value=)rrr) rxrRZget_generic_datarrgatherrrSr) rrNrr=_tasksrrresponserrrrdrdrers<   z"ProxyConfig._update_config_from_db object_typecsZtdd}|dur dSt|tstdt|ddSt|tfdd|DS)aD Check if an object type should be loaded from the database based on general_settings.supported_db_objects. Args: object_type: Type of object to check (e.g., SupportedDBObjectType.MODELS, "models", etc.) Returns: True if the object should be loaded, False otherwise supported_db_objectsNTz(supported_db_objects is not a list, got z. Loading all objects.c3s|] }t|kVqdSrB)r)robjZobject_type_strrdre sz5ProxyConfig._should_load_db_object..) rurrrrxrcrUrr)rrrrdrre_should_load_db_objects  z"ProxyConfig._should_load_db_objectc sZz |jjIdH}W|Sty,}ztdt|g}WYd}~|Sd}~ww)NzQlitellm.proxy_server.py::add_deployment() - Error getting new models from DB - {})r~litellm_proxymodeltable find_manyr\rxrUrqr)rrNrrerdrdre_get_models_from_dbs zProxyConfig._get_models_from_dbc sz@|jddr|j|dIdH}|j||dIdH|jjjddidIdH}|dur6|j|jd IdH|j|dIdHWdSt y^}zt d t |WYd}~dSd}~ww) z{ - Check db for new models - Check if model id's in router already - If not, add to router modelsrrNN)rrqrrur)rz>litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {})rrrr~rrrr_init_non_llm_objects_in_dbr\rxrUrqr)rrNrqrrrerdrdreadd_deployment!s.    zProxyConfig.add_deploymentcs|jddr|j|dIdH|jddr|j|dIdH|jddr.|j|dIdH|jddr=|j|dIdH|jddrJ|IdH|jd drY|j|dIdH|jd drf|IdH|jd dru|j|dIdH|jd dr|j |dIdH|jd dr|j |dIdH|jddr|j |dIdH|jddr|j |dIdH|jddr|j |dIdH|jddrddlm}|j||dIdH|jddr|j|dIdH|jddr|j|dIdHdSdS)z Use this to read non-llm objects from the db and initialize them ex. Vector Stores, Guardrails, MCP tools, etc. rrrNr\ vector_storesvector_store_indexesZmcpZagentsr)r:rtoolsmodel_cost_mapZanthropic_beta_headers sso_settingsZcache_settingsr)CacheSettingsManager)rNrZsemantic_filter_settingsZconfig_overrides)r_init_guardrails_in_db_init_policies_in_db_init_vector_stores_in_db _init_vector_store_indexes_in_db_init_mcp_servers_in_db_init_agents_in_db"_init_pass_through_endpoints_in_db_init_prompts_in_db_init_search_tools_in_db_init_tool_policy_in_db _check_and_reload_model_cost_map(_check_and_reload_anthropic_beta_headers_init_sso_settings_in_db;litellm.proxy.management_endpoints.cache_settings_endpointsrZinit_cache_settings_in_db$_init_semantic_filter_settings_in_db%_init_hashicorp_vault_config_override)rrNrrdrdrerKsR                 z'ProxyConfig._init_non_llm_objects_in_dbc sddl}ddl}ddlm}z|jjjddidIdH}|dus&|jdur)WdS|j}t|t r6| |}| dd}|durCWdSt |drx|j |krx|j|}|rs|D]} t| |rr| jdurr| jjdurrtd WdSqWtd |j|j|| d d r|j|td IdH} | r|j| tdntd||_ WdSty} ztd| WYd} ~ dSd} ~ ww)z Initialize MCP semantic filter settings from database. Called periodically (approximately every 10 seconds) by background task to hot-reload settings across all pods. rNSemanticToolFilterHookrrvrmcp_semantic_tool_filterrz=Semantic filter settings unchanged, skipping reinitializationzYSemantic filter settings unchanged, but hook is missing or uninitialized. Reinitializing.rFrrsz)MCP Semantic Filter reinitialized from DBzMCP Semantic Filter disabledz5Error initializing semantic filter settings from DB: )rrT'litellm.proxy.hooks.mcp_semantic_filterrr~r find_uniquerrrrrrrrXZget_custom_loggers_for_typefilterZ tool_routerrxrSrRZremove_callbacks_by_typer initialize_from_configrsrYrr\rU) rrNrrTr config_recordrvmcp_semantic_filter_configZ active_hooksZ active_hookhookrerdrdrersv             z0ProxyConfig._init_semantic_filter_settings_in_dbc sz<|jjjddidIdH}|dur;|jdd|jdd|jdddd |jD}|j|d WdSWdStyZ}zt d t |WYd}~dSd}~ww) zS Initialize SSO settings from database into the router on startup. rbZ sso_configrNZ role_mappingsZ team_mappingsr<cSsi|] \}}||qSrd)rrrcrrdrdre sz8ProxyConfig._init_sso_settings_in_db..rzHlitellm.proxy.proxy_server.py::ProxyConfig:_init_sso_settings_in_db - {}) r~Zlitellm_ssoconfigr rrrrr\rxrUrqr)rrNrZuppercase_sso_settingsrerdrdrers0   z$ProxyConfig._init_sso_settings_in_dbc sddlm}m}m}m}m}z^|jjjddidIdH}|dus&|j dur2|j dur/||WdS||j }|j |kr?WdS| |} ||} || z|j ddWn t y`|| w||_ tdWdSt y} ztd t| WYd} ~ dSd} ~ ww) z Load Hashicorp Vault config override from DB. Decrypts sensitive fields, sets HCP_VAULT_* env vars, and reinitializes the secret manager. Called periodically via _init_non_llm_objects_in_db to sync config across pods. r)HASHICORP_ENV_VAR_MAPPING_clear_hashicorp_vault_state_get_current_env_values_parse_config_value _set_env_vars config_typeZhashicorp_vaultrN)rz.Hashicorp Vault config override loaded from DBz9Error loading Hashicorp Vault config override from DB: %s).z@Anthropic beta headers config reloaded successfully. Providers: z3Error in _check_and_reload_anthropic_beta_headers: )r~rr rrrr&rxrR"last_anthropic_beta_headers_reloadr(r)r\rc&litellm.anthropic_beta_headers_managerr8r.r/r&sumrrUr)rrNr rrrr0r1r2r3r4rer8rprovider_countrdrdrers        & z4ProxyConfig._check_and_reload_anthropic_beta_headerscCsddlm}||dS)a5 Convert a DB prompt object to a PromptSpec object. Handles the versioning of the prompt, if the DB prompt has a version, it will be used to create the versioned prompt_id. Args: db_prompt: The DB prompt object Returns: The PromptSpec object r)create_versioned_prompt_spec db_prompt)&litellm.proxy.prompts.prompt_endpointsr?)rrAr?rdrdre_get_prompt_spec_for_db_prompts z*ProxyConfig._get_prompt_spec_for_db_promptc sddlm}ddlm}z|jjIdH}|D]}|j|d}|j|dqWdSt yG}zt d t |WYd}~dSd}~ww)Nr)IN_MEMORY_PROMPT_REGISTRY) PromptSpecr@)promptzClitellm.proxy.proxy_server.py::ProxyConfig:_init_prompts_in_db - {})Z%litellm.proxy.prompts.prompt_registryrDZ"litellm.types.prompts.init_promptsrEr~Zlitellm_prompttablerrCZinitialize_promptr\rxrSrqr)rrNrDrEZ prompts_in_dbrFZ prompt_specrerdrdrers"   zProxyConfig._init_prompts_in_dbc sddlm}m}m}z"|j|dIdH}tdt||D] }|jt ||dqWdSt yJ}zt d t|WYd}~dSd}~ww)Nr)IN_MEMORY_GUARDRAIL_HANDLER GuardrailGuardrailRegistryrzguardrails from the DB %s) guardrailzFlitellm.proxy.proxy_server.py::ProxyConfig:_init_guardrails_in_db - {}) Z+litellm.proxy.guardrails.guardrail_registryrGrHrIZget_all_guardrails_from_dbrxrSrZsync_guardrail_from_dbrr\rUrq)rrNrGrHrIZguardrails_in_dbrJrerdrdrer(s. z"ProxyConfig._init_guardrails_in_dbc sddlm}ddlm}z |}|}|j|dIdH|j|dIdHtdWdStyJ}zt d t |WYd}~dSd}~ww)zi Initialize policies and policy attachments from database into the in-memory registries. r)get_attachment_registry)get_policy_registryrNz4Successfully synced policies and attachments from DBzDlitellm.proxy.proxy_server.py::ProxyConfig:_init_policies_in_db - {}) Z/litellm.proxy.policy_engine.attachment_registryrKZ+litellm.proxy.policy_engine.policy_registryrLZsync_policies_from_dbZsync_attachments_from_dbrxrSr\rUrqr)rrNrKrLZpolicy_registryZattachment_registryrerdrdrerCs*    z ProxyConfig._init_policies_in_dbc srddlm}z|}|j|dIdHtdWdSty8}ztdt|WYd}~dSd}~ww)z Initialize tool policy from database into the in-memory registry. Synced periodically by add_deployment -> _init_non_llm_objects_in_db. r)get_tool_policy_registryrNz'Successfully synced tool policy from DBzGlitellm.proxy.proxy_server.py::ProxyConfig:_init_tool_policy_in_db - {}) Z%litellm.proxy.db.tool_registry_writerrMZsync_tool_policy_from_dbrxrSr\rUrqr)rrNrMregistryrerdrdrercs z#ProxyConfig._init_tool_policy_in_dbc ddlm}z/|j|dIdH}t|dkrWdStjdur(||dt_WdS|D] }tjj|dq*WdStyS}zt d t |WYd}~dSd}~ww)NrrUr)r) vector_storeIlitellm.proxy.proxy_server.py::ProxyConfig:_init_vector_stores_in_db - {}) rXrVZ_get_vector_stores_from_dbrrTrTZadd_vector_store_to_registryr\rxrUrqr)rrNrVrrPrerdrdrerus2   z%ProxyConfig._init_vector_stores_in_dbc rO)Nr)VectorStoreIndexRegistryr)r)vector_store_indexrQ) rXrRZ!_get_vector_store_indexes_from_dbrrTZvector_store_index_registryZupsert_vector_store_indexr\rxrUrqr)rrNrRrrSrerdrdrers4    z,ProxyConfig._init_vector_store_indexes_in_dbc s~ddlm}|stddSddlm}z |IdHWdSty>}ztd t |WYd}~dSd}~ww)Nr)is_mcp_availablez.T)reverse) rrDrrDrTrr^rsortedr)rr]rrZ combined_listZ idx_to_deleterOrRrdrdredelete_credentialss   zProxyConfig.delete_credentialsc sz"|jjIdH}fdd|D}|IdHt|WdStyA}ztd t |gWYd}~Sd}~ww)Ncsg|]}|qSrd)r\rrrdrer5sz/ProxyConfig.get_credentials..zSlitellm.proxy_server.py::get_credentials() - Error getting credentials from DB - {}) r~Zlitellm_credentialstablerrarZupsert_credentialsr\rxrUrqr)rrN credentialsrerdrreget_credentials2s&  zProxyConfig.get_credentials)rNrBF)Zr\rr__doc__rrrrrrr rrrrrrr rrrrrrrrrrrrrrTrvrrLrKrHrGrsrurr{rrrrrr rrrrr staticmethodrrrrrrrrrrrrArrrrrrrrrrCrrrrrrrrrrrr\rarcrdrdrdrersh   ('+ % ; 9# Z* */ ; B1 J  "    2 - HKN  T -  "  *?R ;j k 0 rcKsddl}||tjd<dS)Nrrk)rdumpsrr)rrrdrdresave_worker_configHsrhr?csddlm}|tdddkrt|a|a|durBddl}ddl m }m }m }|j |jd|j |jd|j |jd|durjddl}ddl m }m }m }|j |jd|j |jd|j |jdn]|d ur|d urtjd d}|dur|d krddl}dd l m }m }|j |jd|j |jdn)|d krddl}ddl m }m }m }|j |jd|j |jd|j |jdditii}|rtjt|dIdH\aaa| r| a| |td<|r|a||td<|r|tjd<|r||td<|r |a||td<|r|a||td<|r||td<| dur,dt_d|dd<| dur:dt_d|dd<| rF| t_| |dd<t rJ | a!dS)Nr) show_bannerZLITELLM_DONT_SHOW_FEEDBACK_BOXrgrT)verbose_loggerrxry)levelFZ LITELLM_LOGINFOrwDEBUGZgeneralrnrr0ZAZURE_API_VERSION max_tokens temperaturerequest_timeoutalias drop_paramsadd_function_to_promptr)"Z!litellm.proxy.common_utils.bannerrirrSrrr user_model user_debuglogginglitellm._loggingrjrxrysetLevelrlrmrrrrrrsrru user_headers user_api_baseuser_temperatureuser_request_timeoutrTrrrsr experimentaluser_telemetry)r/rqr0 api_versionrSZdetailed_debugrornrprZ telemetryrrrsrsave use_queuerrirvrjrxryZlitellm_log_settingZdynamic_configrdrdrerNs               rc cshtd|D])}td|zdt|dVWqty1dt|dVYqwdS)Ninside generatorzreturned chunk: %sdata: rA)rxrSrrgrr\)rchunkrdrdredata_generators   rr request_datac Cstdzkt|4IdHJ}tj|||dIdH}|2z03dHW}|jdd}z d|dVWqtyO}zdt|dVWYd}~qd}~ww6WdIdHn 1IdHsawYd}d|dVWdSty}zXtd t|tj |||d IdHtd |d t |t r|t|}t t|d |t|d dt|ddt|ddd}td|i} d| dVWYd}~dSd}~ww)Nr)rrrTrrrA[DONE]zTlitellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {}rZoriginal_exceptionrAn error occurred: P Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`r^rUNonerrr^rUrrr)rxrSrBrqasync_post_call_streaming_hookmodel_dump_jsonr\rrUrqpost_call_failure_hookrrr>rrrgr) rrrrcre done_message error_msgproxy_exceptionerror_returnedrdrdreasync_assistants_data_generatorsb   (       rcCs4|d}t|tr |S|d}t|tr|SdS)a Prefer the original client-requested model (pre-alias mapping) when available. Pre-call processing can rewrite `request_data["model"]` for aliasing/routing purposes. The OpenAI-compatible public `model` field should reflect what the client sent. Z_litellm_client_requested_modelr/rg)rrr)rZrequested_modelrdrdre)_get_client_requested_model_for_streamings   rrrequested_model_from_clientmodel_mismatch_loggedc Cs|r t|ttfs ||fSt|tr|dnt|dd}|s0||kr0td|d||d}t|tr=||d<||fSz t|d|W||fStyo}ztj d|d|t |t |ddWYd}~||fSd}~ww)Nr/zolitellm_call_id=%s: streaming chunk model mismatch - requested=%r downstream=%r. Overriding model to requested.litellm_call_idTzPlitellm_call_id=%s: failed to override chunk.model=%r on chunk_type=%s. error=%srw) rrrrrrxrSrEr\rrUr)rrrrZdownstream_modelrerdrdre_restamp_streaming_chunk_models<     rcCsbtdzZzd}t|d}d}d}tj|||d2zp3dHW}tj||||r+|nddIdH}t|ttfrCt j |d}||7}t ||||d\}}t|t rZ|j d d d }nt|trh|d rh|}n&z d |d VWqty} zd t| d VWYd} ~ qd} ~ ww6|dur|Vd } d | d VWnnty } zatdt| tj|| |dIdHtd| dt| tr| t| trt| } nt| } tt| d| t| ddt| ddt| ddd} td| i} d | d VWYd} ~ nd} ~ wwWtjd dBt|dr?z |IdHWn!ty>} z td| WYd} ~ nd} ~ wwWddSWddSWddS1s[wYdStjd d?t|drz |IdHWn ty} z td| WYd} ~ nd} ~ wwWdwWdwWdw1swYw)Nr)rFrgrrr)rrrZ str_so_far)Z response_obj)rrrrT)r exclude_unsetrrArzIlitellm.proxy.proxy_server.async_data_generator(): Exception occured - {}rrrr^rUrrrrrr)shieldaclosez7async_data_generator: error closing response stream: %s) rxrSrrqZ'async_post_call_streaming_iterator_hookrrrJrKrTZget_response_stringrrrrrr\rUrqrrrr>rrrgranyioZ CancelScoperr BaseException)rrr error_messagerrZ _str_so_farrZ response_strrerrrrrdrdreasync_data_generatorNs     $         & 2    rcCst|||dS)Nrrr)rrrdrdreselect_data_generators rr/cCsf|di}|didd}zd|vs|dr |dd}t|}|WSty2iYSw)Nrrr/ZazureZ base_model)rrTget_model_infor\)r/rZmodel_to_lookuplitellm_model_infordrdreget_litellm_model_infos    rcCstd|ddS)Nz$Backing off... this was attempt # %stries)rxrS)rrdrdre on_backoffsrcCsft|tot|ddduot|jtod|jv }tddur"dS|r1tt dt|d|S)Nr^z"Max parallel request limit reachedZ1disable_retry_on_max_parallel_request_limit_errorTgiveup)eventrU) rr>rr^rrurrxrSrrg)reresultrdrdrers  rc @sdeZdZedeededeefddZe de deefddZ edeed e e effd d Zede d eed efddZede fddZede d ed eddfddZeddZeddZede d ededededef dd Zed!efd"d#Zed!ede ded efd$d%Zed&ee ded edeefd'd(Zed)d*Zed+d,ZdS)-rrsrqrtcCst|j||ddS)z*Initialize logging and alerting on startup)rsrtN)rZZ startup_event)clsrsrqrtrdrdrers  z-ProxyStartupEvent._initialize_startup_loggingrucCsFddlm}|dd}t|tr||}|r|dur!tddSdS)z Validates that when use_redis_transaction_buffer is enabled, a Redis cache is properly configured in litellm_settings. r)rZuse_redis_transaction_bufferFNa`use_redis_transaction_buffer` is enabled in general_settings but no Redis cache is configured. This will cause spend updates to not be tracked. Add a Redis cache in litellm_settings: litellm_settings: cache: true cache_params: type: redis url: os.environ/REDIS_URL )litellm.secret_managers.mainrrrrr)rurtrZ_use_redis_transaction_bufferrdrdrers   z;ProxyStartupEvent._validate_redis_transaction_buffer_configrvcsddlm}|dd}|r|ddstddStd|dud ||j||d IdH}|rBtd tj|dSt d dS) z1Initialize MCP semantic tool filter if configuredrrrNrFzKSemantic tool filter not configured or not enabled, skipping initializationz.Initializing semantic tool filter: llm_router=z , config=rz$Semantic tool filter hook registeredzASemantic tool filter hook was configured but failed to initialize) rrrrxrSr rTrXrYrc)rrsrvrr rrdrdrers6    z2ProxyStartupEvent._initialize_semantic_tool_filterrNrrcCsx|dddur/|dD]\}}t|tr$|dr$t||d|<qtdi|d}nt}tj|||ddS)zInitialize JWT auth on startuplitellm_jwtauthNr)rNrrrrd) rrrrrrr8rVr)rrurNrrrrrrdrdrer;s z&ProxyStartupEvent._initialize_jwt_authrycCsJtjdur tdttdd|dgiidtjtjdtjtjdd dS)z Adds a global proxy budget to dbNzPbudget_duration not set on Proxy. budget_duration is required to use max_budget.userrZ update_data)rbudget_duration) request_typerr\durationraliasesrrerrZ query_typeZupdate_key_values)rTrr\rrrr)rryrdrdrerPs, z)ProxyStartupEvent._add_proxy_budget_to_dbrzrNc sZzd|}t|||dIdHWdSty,}z td|WYd}~dSd}~ww)zSWarm global spend cache once at startup to reduce impact of first wave of requests.rs)r|rrrNNz;Global spend cache warm-up at startup skipped or failed: %s)rqrr\rxrS)rrzrrrNr|rerdrdrerms z*ProxyStartupEvent._warm_global_spend_cachecsxtjdurdStjdpg}|r8tdd|Dr:ddlm}dd|D}||tttd d IdHdSdSdS) z%Update the default team member budgetNteamscss|]}t|tVqdSrB)rrrrrdrdrerszGProxyStartupEvent._update_default_team_member_budget..r)!update_default_team_member_budgetcSrr)ZNewUserRequestTeamrrdrdrerrzHProxyStartupEvent._update_default_team_member_budget..ra)rr) rTZdefault_internal_user_paramsrr57litellm.proxy.ui_crud_endpoints.proxy_setting_endpointsrrErrC)rZ_teamsrZteams_pydantic_objrdrdrers  z4ProxyStartupEvent._update_default_team_member_budgetc szSddl}ddlm}tjjjddidIdH}|rL|jrO|j}t|t r+| |nt |fdd|D}|rRt |td t|WdSWdSWdSWdStym}z td |WYd}~dSd}~ww) z Load persisted UI settings from the database and sync runtime flags into general_settings so they take effect immediately after startup. rN)_RUNTIME_GENERAL_SETTINGS_FLAGSrb ui_settingsrcsi|] }|vr||qSrdrdr:rrdrers zKProxyStartupEvent._sync_ui_settings_to_general_settings..z5Synced UI settings to general_settings on startup: %sz1UI settings sync on startup skipped or failed: %s)rrrrNr~Zlitellm_uisettingsr rrrrrrur=rxrRrrr\rS)rrrrrawZ flags_to_syncrerdrrers6      z7ProxyStartupEvent._sync_ui_settings_to_general_settingsr{r|r}c s|ddlm}ddlm}ttttdd|id|idda|t dt d||} |t dd } | d d d urNt ||d } tj| jd | ddtdtjtd | |t|gddtd| dd d uruddlm} t| |t|dtdtp{tatdur|durz5|jjjddidIdH} | durt| jtr| j d}|dust|tr|dkrdat !dWnt"y}zt #dt|WYd}~nd}~wwtdur tjt$j%d d||gddtdt$j%||dIdHtjt$j&d d|gd dtdt$j&|d!IdH|j't|||d"IdH|j(td#IdH| d$durt)}| d%}|rfdd&l*m+}z|,|}tj|j-||gd'dtd(t !d)|WnBt.yet /d*|Yn1w| d+d,}zt0|}tj|j-d |t dd-|gd'dtdWnt.yt /d.Ynwt1durz&dd/l2m3}|||t1d0}tj|j4d t5t ddd1dtdt !d2Wn t"y}zt #d3|t #d4WYd}~nd}~wwt1dur.z&dd5l6m7}|||t1d0}tj|j8d t5t ddd6dtdt !d7Wn t"y-}zt #d8|t #d9WYd}~nd}~wwtj9d d:t !d;tdS)'r~start_token_refresh_taskZ&DISABLE_PRISMA_HEALTH_CHECK_ON_STARTUPFTstart_db_health_watchdog_task)rr\connectrrxrSrr~rrrZcheck_view_existsZ(_set_spend_logs_row_count_in_proxy_staterZ health_checkrrZhandle_db_exception)rrprqrrrNrerdrdrer]sd     z&ProxyStartupEvent._setup_prisma_clientcCs^ddlm}m}|rddl}|jddd|r-ddlm}|}|t ddSdS) z Initialize dd tracer - if `USE_DDTRACE=true` in .env DD tracer is used to trace Python applications. Doc: https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/python/ r)_should_use_dd_profiler_should_use_dd_tracerNTF)rvopenai)ProfilerzDatadog Profiler started......) Z%litellm.litellm_core_utils.dd_tracingrrddtraceZ patch_allZddtrace.profilingrrrxrS)rrrrrZprofrdrdrers z!ProxyStartupEvent._init_dd_tracerc Cs>tdds tddSzddl}td}|stdtd}|s'td i}td p2td }|r9||d <td }|||rD|ndd}|durgz tt||d<Wntt fyftd|w|j di|d|d|d|d}d|vr|d|d7}t |WdSt yt dYdSw)a Optional continuous profiling via Grafana Pyroscope. Off by default. Enable with LITELLM_ENABLE_PYROSCOPE=true. Requires: pip install pyroscope-io (optional dependency). When enabled, PYROSCOPE_SERVER_ADDRESS and PYROSCOPE_APP_NAME are required (no defaults). Optional: PYROSCOPE_SAMPLE_RATE (parsed as integer) to set the sample rate. ZLITELLM_ENABLE_PYROSCOPEFzWLiteLLM: Pyroscope profiling is disabled (set LITELLM_ENABLE_PYROSCOPE=true to enable).NrZPYROSCOPE_APP_NAMEzsLITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_APP_NAME is not set. Set PYROSCOPE_APP_NAME when enabling Pyroscope.ZPYROSCOPE_SERVER_ADDRESSzLITELLM_ENABLE_PYROSCOPE is true but PYROSCOPE_SERVER_ADDRESS is not set. Set PYROSCOPE_SERVER_ADDRESS when enabling Pyroscope.ZOTEL_ENVIRONMENT_NAMEZLITELLM_DEPLOYMENT_ENVIRONMENT environmentZPYROSCOPE_SAMPLE_RATE)app_nameserver_addressrZ sample_ratez-PYROSCOPE_SAMPLE_RATE must be a number, got: z/LiteLLM: Pyroscope profiling started (app_name=z, server_address=zA). View CPU profiles at the Pyroscope UI and select application 'z'.z sample_rate=zLiteLLM: LITELLM_ENABLE_PYROSCOPE is set but the 'pyroscope-io' package is not installed. Pyroscope profiling will not run. Install with: pip install pyroscope-iord)rrxrS pyroscoperrSrrrlr configurerR ImportErrorrc) rrrrrenv_nameZsample_rate_envZconfigure_kwargsmsgrdrdrersf       z!ProxyStartupEvent._init_pyroscope)r\rr classmethodr rvrr{rrfrrrrrrrrzrrrrrrrrirrrrrrdrdrdrers    (    " ! L : B rz /v1/modelszmodel management) dependenciesrz/modelsreturn_wildcard_routesinclude_model_access_groupsonly_model_access_groupsinclude_metadata fallback_typescopec s|ddlm}ddlm} m} |dur |dkr tdd|dd } |dkr1||tttd IdH} | rt durr)rrr/rrr/rer0Z_text_responser5r2r3r4 _responser1rrdrdre completionsB)                         r:z/v1/embeddings embeddings)rresponse_classrz /embeddingsz /engines/{model:path}/embeddingsz+/openai/deployments/{model:path}/embeddingsc szi}zt|dIdH}tdurtjng}d|vrt|dtrt|ddkrt|ddtrt|dddtrtdur|d|vrtj|dd}|dur|dip[i}|ddt j vpqt fd d t D}|sg} |dD] } | t jd | d qz| |d<|dur|d duri|d <t|dr|jdur|j|d d<t|dr|jdur|j|d d<t|dr|jdur|j|d d<t|dr|jdur|j|d d<t|d} | jd'id|d|d|dddtdtdtdtdtd|d td!td"td#td$td%tIdH} | WSt y<} zt|d} | j!| |ttd&IdHd} ~ ww)(a Follows the exact same API spec as `OpenAI's Embeddings API https://platform.openai.com/docs/api-reference/embeddings` ```bash curl -X POST http://localhost:4000/v1/embeddings -H "Content-Type: application/json" -H "Authorization: Bearer sk-1234" -d '{ "model": "text-embedding-ada-002", "input": "The quick brown fox jumps over the lazy dog" }' ``` rNinputrr/)Zmodel_group_namerrgc3s|]}|VqdSrB)r)rr litellm_modelrdrers  zembeddings..z gpt-3.5-turbo)r/tokensrr\rr^rrrrrrrrrZ aembeddingrqrsrurrrtr{r|rrzr)rerrqrrd)"rrs model_namesrrrrrrrTZopen_ai_embedding_modelsrr!rdecoderr\r^rrrr'rqrurrrtr{r|rrzrr\r.)rrr/rrrouter_model_namesrrZsupports_token_arraysZ input_listir/rrerdr>rer;Os0                  z/v1/moderations moderationsz /moderationsc si}z|IdH}t|}t||t|ttdIdH}tddp)tp)|d|d<tr2t|d<t j ||ddIdH}t t |dt tdIdH}|IdH}tt j|d d d d t|d ipfi}|ddpnd }|ddpvd } |ddp~d } |jtj||| | tt|dd ||d|WSty} zRt j|| |dIdHtdt| t| trtt| dt| t| ddt| ddt| dtj dt| } tt| d| t| ddt| ddt| dddd} ~ ww)a~ The moderations endpoint is a tool you can use to check whether content complies with an LLM Providers policies. Quick Start ``` curl --location 'http://0.0.0.0:4000/moderations' --header 'Content-Type: application/json' --header 'Authorization: Bearer sk-1234' --data '{"input": "Sample text goes here", "model": "text-moderation-stable"}' ``` Nrrrurrrmoderation_modelr/Z moderationrr call_typeZ amoderationrrrsrtrrgrrr'_hidden_paramsrmr|r0allowed_model_regionrrmr|r0r model_regionr hidden_paramsrz@litellm.proxy.proxy_server.moderations(): Exception occured - {}r^rUrrrrr)!bodyorjsonrrrurrrrtrq pre_call_hookrBrrsrrupdate_request_statusrrr=rget_custom_headersr\rrxrUrqrrrr>r'r-) rrrrrQllm_callrrPrmr|r0rerrdrdrerEs                 r9cCs2|jtdIdH}|2z 3dHW}|Vq 6dS)N) chunk_size)Z aiter_bytesr)r9Z _generatorrrdrdre_audio_speech_chunk_generatorYs rXz/v1/audio/speechZaudioz /audio/speechc si}z|IdH}t|}t||t|ttdIdH}|dddur/|jdur/|j|d<t r5t |d<t j ||ddIdH}t |dt t dIdH}|IdH}tt j|dd d d t|d ipei}|d dpmd }|ddpud } |ddp}d } |ddpd } |ddpd } tj||| | t| t|dd d| ||d } d}|dd }|r|}d|vrd|vsd|vrd}tt||| dWSty}zt j|||dIdHtdt|tt|d}~ww)z` Same params as: https://platform.openai.com/docs/api-reference/audio/createSpeech NrFrr/ZaspeechrHrJrrgrrKrLrmr|r0r_rM) rrmr|r0rr_rOZ!fastest_response_batch_completioncall_idrrPz audio/mpeggeminiZttsz preview-ttsz audio/wav)r$rrzAlitellm.proxy.proxy_server.audio_speech(): Exception occured - {}) rQrRrrrurrrr\rtrqrSrrsrrrTrrrUrr1rXr\rrxrrqrrSrtru)rrrrrQrVrrPrmr|r0r_rZcustom_headersr$ request_modelZrequest_model_lowerrerdrdre audio_speeches         r\z/v1/audio/transcriptionsz/audio/transcriptions.rbcsi}zt|IdH}dd|D}t||t|ttdIdH}|dddur4|jdur4|j|d<tddpAtpA|dd|d<trJt|d<t durQt j ng}|j durat dt jd d d t|||d |IdH}t|}|j |_||d <z2ztj||d dIdH}t|dt tdIdH} | IdH} Wnty} z| d} ~ wwW|n|wttj|ddddt| dipi} | ddpd} | ddpd}| ddpd}| ddpd}| ddpd}| dipi}|jtj d(|| ||t|t|dd||| d || WSty}} zYtj!|| |dIdHt"#d$t%| t&| t'rWt t| dt%| j(t| d d!t| d"d!t| d#t jd$t%| }t t| d|t| d d!t| d"d!t| d%dt| d#d&d'd} ~ ww))zq Same params as: https://platform.openai.com/docs/api-reference/audio/createTranscription?lang=curl NcSsi|] \}}|dkr||qS)rbrdrrdrdrerz(audio_transcriptions..rFrrGr/z.File name is None. Please check your file name bad_requestrb)r^rrUr)rrbrCZ transcriptionrHZatranscriptionrJrrgrrKrLrmr|r0r_additional_headersrM) rrmr|r0rr_rOrYrrPrzHlitellm.proxy.proxy_server.audio_transcription(): Exception occured - {}r^rUrrrrrrr8rd))rrrrurrrr\rtrsrAr`r>r'r-rreadioBytesIOrrqrSrr\rWrrrTrrr=rrUrrxrUrqrrrrA)rrrbrrZ form_datarCZ file_contentZ file_objectrVrrerPrmr|r0r_rr_rrdrdreaudio_transcriptionss                      rcz/vertex_ai/livezGOptional model name, used to determine Vertex region for global models.)rzCOverride the Vertex AI project id used for the upstream connection.z;Override the Vertex AI region (for example, 'us-central1'). websocketvertex_projectvertex_locationcst|||||dIdHS)z Vertex AI Live API WebSocket Pass-through Endpoint This endpoint delegates to the WebSocket function defined in llm_passthrough_endpoints.py rdr/rerfrNrrgrdrdre#vertex_ai_live_passthrough_endpoint{s rh)maxsizeintentcCs(d|fg}|dur|d|ft|S)z Build a hashable representation of the realtime query params so we can cache the repetitive model/intent combinations. r/Nrj)rtuple)r/rjparamsrdrdre_realtime_query_params_templates rmz /realtimez'The intent of the websocket connection.zAComma-separated list of guardrail names to apply to this request.rcs>dd|jdp ddD}i}|r|d|d<|jdi|IdHtttt|}||d}|rDd d|dD|d <t|j d pLg} t } | | d <t | d } |j | _fd d} | | _t|d} z| j| t|ttttttttdd IdH\}}WnAty}z5tdz|t ddt!|ddIdHWn tyYnw|j"dddIdHWYd}~dSd}~wwz||d<t#|dt$tdIdH}|IdHWdSt%j&j'y}ztd|j"|j(ddIdHWYd}~dSd}~wtytd|j"dddIdHYdSw)NcSg|] }|r|qSrdr)rprdrdrers z/realtime_websocket_endpoint..zsec-websocket-protocolrgrhrZ subprotocol)r/rdrcSrnrdro)rgrdrdrerrr)rcs tSrBrHrdrwrdre return_bodysz0realtime_websocket_endpoint..return_bodyrZ _arealtime) rrurrrqrrtr{r|rrzr/rzRealtime pre-call errorrZguardrail_error)rUr^)rUrizPre-call error)rreasonrrJzInvalid status codezInternal server errorrd))rrracceptrr rrmrrrXrr!r_urlrQrZ common_processing_pre_call_logicrurrqrrtr{r|rrzr\rxrUZ send_textrrgrrWrrs websockets exceptionsZInvalidStatusCoder)rdr/rjrrZrequested_protocolsZ accept_kwargsrrZ headers_listrrrsr/rrerVrdrwrerealtime_websocket_endpoints         $ ryz/v1/assistantsZ assistantsz /assistantsc si}zs|IdHt||t|ttdIdH}tdur&tddtjj idtj di|IdH}t t j|dddd t|d ipFi}|d dpNd}|d dpVd}|d dp^d}|jtj||||tt|dd||d|WSty} z_t j|| |dIdHtdt| ttt| trtt| dt| j t| ddt| ddt| dt!j"dt| } tt| d| t| ddt| ddt| ddt| dddd} ~ ww)z Returns a list of assistants. API Reference docs - https://platform.openai.com/docs/api-reference/assistants/listAssistants NrFrrr@rrgrrKrLrmr|r0rMrNrzClitellm.proxy.proxy_server.get_assistants(): Exception occured - {}r^rUrrrrrr8rd)#rQrrurrrsrr) no_llm_routerrZaget_assistantsrrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r- rrrrrrPrmr|r0rerrdrdreget_assistants+s                 r|c si}zx|IdH}t|}t||t|ttdIdH}tdur+tddt j j idtj di|IdH}t tj|dddd t|d ipKi}|d dpSd}|d dp[d}|d dpcd} |jtj|||| tt|dd||d|WSty} z^tj|| |dIdHtdt| ttt | trt!t| dt| j"t| ddt| ddt| dt#j$dt| } t!t| d| t| ddt| ddt| dt| dddd} ~ ww)z~ Create assistant API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant NrFrrr@rrgrrKrLrmr|r0rMrNrzElitellm.proxy.proxy_server.create_assistant(): Exception occured - {}r^rUrrrrrrd)%rQrRrrrurrrsrr)rzrZacreate_assistantsrrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r-) rrrrrQrrPrmr|r0rerrdrdrecreate_assistants                r}z"/v1/assistants/{assistant_id:path}z/assistants/{assistant_id:path} assistant_idc i}znt||t|ttdIdH}tdurtddtjjidtj dd|i|IdH}t t j |ddd d t|d ipAi}|d dpId}|d dpQd}|ddpYd} |jtj|||| tt|dd||d|WSty} z^t j|| |dIdHtdt| ttt| trtt| dt| jt| ddt| ddt| dt j!dt| } tt| d| t| ddt| ddt| dt| dddd} ~ ww)z~ Delete assistant API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant rFNrrr@r~rrgrrKrLrmr|r0rMrNrzElitellm.proxy.proxy_server.delete_assistant(): Exception occured - {}r^rUrrrrrrd)"rrurrrsrr)rzrZadelete_assistantrrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r-) rr~rrrrrPrmr|r0rerrdrdredelete_assistants               rz /v1/threadsz/threadsc si}zs|IdHt||t|ttdIdH}tdur&tddtjj idtj di|IdH}t t j|dddd t|d ipFi}|d dpNd}|d dpVd}|d dp^d}|jtj||||tt|dd||d|WSty} z^t j|| |dIdHtdt| ttt| trtt| dt| j t| ddt| ddt| dt!j"dt| } tt| d| t| ddt| ddt| dt| dddd} ~ ww)zs Create a thread. API Reference - https://platform.openai.com/docs/api-reference/threads/createThread NrFrrr@rrgrrKrLrmr|r0rMrNrzClitellm.proxy.proxy_server.create_threads(): Exception occured - {}r^rUrrrrrrd)#rQrrurrrsrr)rzrZacreate_threadrrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r-r{rdrdrecreate_threadsPs               rz/v1/threads/{thread_id}z/threads/{thread_id} thread_idc r)zs Retrieves a thread. API Reference - https://platform.openai.com/docs/api-reference/threads/getThread rFNrrr@rrrgrrKrLrmr|r0rMrNrz?litellm.proxy.proxy_server.get_thread(): Exception occured - {}r^rUrrrrrrd)"rrurrrsrr)rzrZ aget_threadrrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r- rrrrrrrPrmr|r0rerrdrdre get_thread               rz /v1/threads/{thread_id}/messagesz/threads/{thread_id}/messagesc si}zz|IdH}t|}t||t|ttdIdH}tdur+tddt j j idtj dd|i|IdH}t tj|ddd d t|d ipMi}|d dpUd}|d dp]d} |ddped} |jtj||| | tt|dd||d|WSty} z^tj|| |dIdHtdt| ttt | trt!t| dt| j"t| ddt| ddt| dt#j$dt| } t!t| d| t| ddt| ddt| dt| dddd} ~ ww)zv Create a message. API Reference - https://platform.openai.com/docs/api-reference/messages/createMessage NrFrrr@rrrgrrKrLrmr|r0rMrNrzAlitellm.proxy.proxy_server.add_messages(): Exception occured - {}r^rUrrrrrrd)%rQrRrrrurrrsrr)rzrZ a_add_messagerrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r- rrrrrrQrrPrmr|r0rerrdrdre add_messages s                rc r)z Returns a list of messages for a given thread. API Reference - https://platform.openai.com/docs/api-reference/messages/listMessages rFNrrr@rrrgrrKrLrmr|r0rMrNrzAlitellm.proxy.proxy_server.get_messages(): Exception occured - {}r^rUrrrrrrd)"rrurrrsrr)rzrZ aget_messagesrrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r-rrdrdre get_messagess rrz/v1/threads/{thread_id}/runsz/threads/{thread_id}/runsc si}z|IdH}t|}t||t|ttdIdH}tdur+tddt j j idtj dd|i|IdH}d|vrR|ddurRt t|||d d id IdHWSttj|d d ddt|dipgi}|ddpod }|ddpwd } |ddpd } |jtj||| | tt|dd ||d|WSty} z^tj|| |dIdHtdt| tt !t"| trt#t| dt| j$t| ddt| ddt| dt%j&dt| } t#t| d| t| ddt| ddt| dt| dddd} ~ ww) zi Create a run. API Reference: https://platform.openai.com/docs/api-reference/runs/createRun NrFrrr@rrTrr") generatorr$rrrgrrKrLrmr|r0rMrNrz?litellm.proxy.proxy_server.run_thread(): Exception occured - {}r^rUrrrrrrd)'rQrRrrrurrrsrr)rzrZ arun_threadrrrrrqrTrrrr=rrUr\rrxrrqrrSrtrurr>rAr'r-rrdrdre run_thread s                 r)BaseTokenCounterr model_to_usecCs|durdSddlm}|didd}d}d}zL|||did|did|did d \}}}}dd lm}dd lm} ||} | ra| j|| d } | durd| ||fWSWdSWdSt y| dsxd| vrddl m } | } | ||fYSYdSw)z Auto-route to the correct provider's token counter based on model/deployment. Uses the existing get_provider_model_info infrastructure with switch-case pattern. Nr)rrr/rgr!r0api_key)r/r!r0r) LlmProviders)ProviderConfigManager)r/rz anthropic/Z anthropic)AnthropicModelInfo)NNN)Z1litellm.litellm_core_utils.get_llm_provider_logicrrr?rrZrZget_provider_model_infoZget_token_counterr\rrZ#litellm.llms.anthropic.common_utilsr)rrrZ full_modelr/r!Zdynamic_api_keyr0rrZllm_provider_enumZprovider_model_inforZanthropic_model_infordrdre_get_provider_token_counterK!sN     rz/utils/token_counterz llm utils)rrresponse_model call_endpointc sBddlm}|j}|j}|j}|j}|j}|dur(|dur(|dur(tdddd}d} d} tdurNz tj |j idIdH}Wnt yMt dYnw|duru|d id } |d i} t|d id | vru| d d d } | py|j } d} d} |dur|durt|| \} }} |dur|} | dur| j| ddur| j| pd||||j ||dIdH}|dur|jdurtjdurt|jpddd |jpddt d|jd|jdn|dur|Stjdurtddd ddd}| durttt| dd}tjj| |d}t|d}|| |||d}t ||j | |d S)!z Args: request: TokenCountRequest call_endpoint: bool - When set to "True" it will call the token counting endpoint - e.g Anthropic or Google AI Studio Token Counting APIs. Returns: TokenCountResponse r) token_counterNrz/prompt or messages or contents must be providedr@)r/Zrequest_kwargszVlitellm.proxy.proxy_server.token_counter(): Exception occured while getting deploymentrr/rrriTr!rg)rrhcontentsrr[rsystemzToken counting failedZtoken_counting_errorrrz Provider token counting failed (z): z". Falling back to local tokenizer.z?Token counting is disabled and no provider API result availableZtoken_counting_disabledicustom_tokenizer)r/rrU)r/r7rhr)r&r[Z model_usedZtokenizer_type)!rTrrFrhrrrrrsZasync_get_available_deploymentr/r\rxrUrrOrrZshould_use_token_counting_apiZ count_tokensrZdisable_token_counterr>rrrcrr rr)Z_select_tokenizerrrM)rrrrFrhrrrrrPrrZprovider_counterr!_modelrrZ_tokenizer_usedZtokenizer_usedr&rdrdrer!s          rz/utils/supported_openai_params)rrc sPztj|d\}}}}dtj||diWSty'tddd|idw)a* Returns supported openai params for a given litellm model name e.g. `gpt-4` vs `gpt-3.5-turbo` Example curl: ``` curl -X GET --location 'http://localhost:4000/utils/supported_openai_params?model=gpt-3.5-turbo-16k' --header 'Authorization: Bearer sk-1234' ``` rwsupported_openai_paramsr/r!rrzCould not map model={}r@)rTrZget_supported_openai_paramsr\rrq)r/r!rrdrdrer"s rz/utils/transform_requestcsddlm}||j|jdS)Nr)return_raw_request)endpointr()rZrrI request_body)rrrdrdretransform_request1"s rrcs|durtddtjjidg}|D]-}|didd}|dur$q|jjjd|idIdH}|dur@|j|j kr@| |q|S) Check if model is in db Check if db model is 'created_by' == user_api_key_dict.user_id Only return models that match Nrrr@rrbrmr) rr)db_not_connected_errorrrr~rr rqr\r)rrrNfiltered_modelsr/rbrnrdrdre_check_if_model_is_user_added="s&     ruser_rowcCsHg}|D]}|didd}|dur!||jvr!|tt|q|S)zm Check if model is a team model Check if user is a member of the team that the model belongs to rr^N)rrrrr)rrZuser_team_modelsr/Z model_team_idrdrdre_check_if_model_is_team_model\"s  rr cs|durtddtjjidt|||dIdH}|jrHz|jjjd|jidIdH}Wnt y;tddd idw|t | pCg|d 7}t |d }|S) rNrrr@)rrrNr\rrzUser not found)rr)r) rr)rrrr\r~litellm_usertabler r\rrMr)r rsrrNr unique_modelsrdrdrenon_admin_all_modelsp"s2      rteam_db_objects_typedc Csi}|D]}t|jdkstjj|jvrX|}|durW|D]8}|didd}|dur/q|didd}d}|durBd}n||jvrId}|rV||t  |jqq|jD],} |j| |jd} | dur| D]}|didd}|dur||t  |jqkq[q|S) z' Add team models to all models rNrrbr^FTr.r^) rrr@all_proxy_modelsrrMrr^rradd) rrsr team_objectrr/rm team_model_id can_add_modelr._modelsrdrdre_add_team_models_to_all_models"sB   r user_teamsc sg}|dkr|jjIdH}dd|D}n|jjjdd|iidIdH}dd|D}t||d }i}|D] \}}t|||<q:|S) z Get all models across all teams user is in. 1. Get all teams user is in 2. Get all models across all teams 3. Return {"model_id": ["team_id1", "team_id2"]} rNcSg|] }tdi|qSrr9rrteam_db_objectrdrdrer"z'get_all_team_models..r^rrcSrrrrrdrdrer"r)rrs)r~litellm_teamtablerrrr) rrNrsrZteam_db_objectsrZreturned_team_modelsrmteam_idsrdrdreget_all_team_models"s*   ruser_db_objectcCsXg}|jD]$}|j|d}|dur)|D]}|didd}|dur(||qq|S)z7 Get all models that user has direct access to r.Nrrb)rrMrr)rrsdirect_access_modelsr/Z deploymentsrrmrdrdreget_direct_access_models"s   rcsfd}g}|jtjkrd}|jdd}n+|jdur?|jjjd|jidIdH}|dur?tdi| }|j p8g}t ||d}|durt |||dIdH}|D]=} | d i d d} | d i d d} | durd } | dur~| | g} | r}| | vr}d} nd} | r| | g| d d <qO|D]} | d i d d} | dur| |vrd| d d<qdd|D}|S)z5 Get all models across all teams user is in. NrT)Zexclude_team_modelsr\r)rrs)rrNrsrrbr^Faccess_via_team_ids direct_accesscSs8g|]}|didds|didgr|qS)rrFrr)rrrdrdrerD#sz9get_all_team_and_direct_access_models..rd) user_roler; PROXY_ADMINrzr\r~rr r:rrrrr)rrNrsr rrrZ user_objectrrrmZteam_only_model_idZ can_use_modelrrdrdre%get_all_team_and_direct_access_models #sd          rrSc CsH|di}|dur%d}|dur|j|iddpd}nd}t|}||d<t|d }|ikrN|d i}|d d}ztj|d }Wn tyMi}Ynw|ikr|d i}|d d}|r|d } t| d kro| d}z tj|| d d}Wn tyi}Ynw| D] \} } | |vr| || <q||d<t |dhd}|S)a Enrich a model dictionary with litellm model info (pricing, context window, etc.) and remove sensitive information. Args: model: Model dictionary to enrich debug: Whether to include debug information like openai_client llm_router: Optional router instance for debug info Returns: Enriched model dictionary with sensitive info removed rTrNasync)rr(Z client_typeZllm_router_is_None openai_clientrwrr/rrrrlitellm_credential_name) excluded_keys) rZ _get_clientrrrTrr\rrrr) r/rSrsrZ_openai_clientrrrr? split_modelrrrdrdre$_enrich_model_info_with_litellm_dataM#sT          rsearchpagesizersort_byc s|r|s |dfS|fdd|D}g}t} |D]"} | di} | dd} | d} | r>| r>| | q!|| q!t|}t| }||}g}d}||}|durzvd d d i}| rpd d t| ii|d<|jj j |dIdH}||}|r|dkr|jj j ||dIdH}|D]}| |g}|r| |qn-||krt|||}|dkr|jj j ||dIdH}|D]}| |g}|r| |qWn ty}ztdt||}WYd}~nd}~ww|}||}||fS)aC Apply search filter to models, querying database for additional matching models. Args: all_models: List of models to filter search: Search term (case-insensitive) page: Current page number size: Page size prisma_client: Prisma client for database queries proxy_config: Proxy config for decrypting models sort_by: Optional sort field - if provided, fetch all matching models instead of paginating at DB level Returns: Tuple of (filtered_models, total_count). total_count is None if not searching. Ncs$g|]}|ddvr|qS)r.rg)rrr search_lowerrdrer#sz2_apply_search_filter_to_models..rrnFrbrr.Z insensitive)containsmodenotrrmr)rZtakez,Error querying database models with search: )rrrrrrrrr~rcountrrrrr\rxrUr)r rrrrNrrZfiltered_router_modelsZfiltered_config_modelsZdb_model_ids_in_routerrrZ is_db_modelrmZconfig_models_countZdb_models_in_router_countZrouter_models_countrvZdb_models_total_countZmodels_needed_for_pageZdb_where_conditionsearch_total_countZ db_models_rawrndecrypted_modelsZmodels_to_fetchrerrdrre_apply_search_filter_to_models#s               rdtc Cs|durdSt|trBz*|dr|ddn|}t|}|jdur-|jtjd}|WS| tj}|WSt t fyAYdSwt|trY|jdurS|jtjdS| tjSdS)a Normalize a datetime value to a timezone-aware UTC datetime for sorting. This function handles: - None values: returns None - String values: parses ISO format strings and converts to UTC-aware datetime - Datetime objects: converts naive datetimes to UTC-aware, and aware datetimes to UTC Args: dt: Datetime value (None, str, or datetime object) Returns: UTC-aware datetime object, or None if input is None or cannot be parsed NZz+00:00tzinfo) rrr!rrr(rrutc astimezonerr)rZdt_strZ parsed_dtrdrdre_normalize_datetime_for_sorting$s&       rasc sort_orderc srdvr|S|dkdtttfdtffdd }z t||d}|WStyI}ztdd t||WYd }~Sd }~ww) a Sort models by the specified field and order. Args: all_models: List of models to sort sort_by: Field to sort by (model_name, created_at, updated_at, costs, status) sort_order: Sort order (asc or desc) Returns: Sorted list of models )r.rorpcostsr'descr/rc s$|di}dkr|ddSdkr7|d}t|}|dur5s-tjjtjdStjjtjdS|Sdkr\|d}t|}|durZsRtjjtjdStjjtjdS|Sdkr|dd pgd }|d d pod }||}|d krs~t d St d S|Sd kr|dd}|SdS)Nrr.rgrorrprZinput_cost_per_tokenrZoutput_cost_per_tokenrdz-infr'rnF) rrrrmaxrrrrrl) r/rroZ normalized_dtrpZ input_costZ output_costZ total_costrnr_rrdre get_sort_keyc$sD    z"_sort_models..get_sort_key)rcr_zError sorting models by rZN)rrrrr`r\rxrU)r rrrZ sorted_modelsrerdrre _sort_modelsH$s   -r total_countc Cs||durt|}|d|}|dkr| | nd}||||}td|d|d|d|d| |||||d S) aD Paginate models and return response dictionary. Args: all_models: List of all models page: Current page number size: Page size total_count: Total count (if None, uses len(all_models)) search: Search term (for logging) Returns: Paginated response dictionary NrirzPagination: skip=z, take=z, total_count=z, total_pages=z , search=rrZ current_page total_pagesr)rrxrS)r rrrrskiprZpaginated_modelsrdrdre_paginate_models_response$s  rc sz(|jjjd|idIdH}|durtd|dgWStdi|}Wn!tyJ}ztd|dt |gWYd}~Sd}~wwt }t |j dks\t jj|j vr|rb|ng}|dur|D]1} | d id d} | dur{qj| d idd} d } | durd } n| |krd } | r|| qjn-|j D])} |r|j| |d ng}|dur|D]} | d id d} | dur|| qqz+|j rt jj|j vr|jjjdd|j iidIdH}|D] }|j} | r|| qWn ty}ztd|dt |WYd}~nd}~wwg}|D]B}|d i}|d d} |dd r6||q|dg}t|trN||vrN||q| r[| |vr[||q|S)a Filter models by team ID. Returns models where: - direct_access is True, OR - team_id is in access_via_team_ids Also searches config and database for models accessible to the team. Args: all_models: List of models to filter team_id: Team ID to filter by prisma_client: Prisma client for database queries llm_router: Router instance for config queries Returns: Filtered list of models r^rNzTeam z not found in databasezError fetching team rZrrrbFTrr.rz(Error querying database models for team rrrd)r~rr rxrcr9rr\rUrrrrr@rrrMrrrrrmrSrrr)r r^rNrsrrreZteam_accessible_model_idsrr/rmrrr.rrvrnrrrrrdrdre_filter_models_by_team_id$s             rc s d}|dur|j|d}|rt|}|durUz|jjjd|idIdH}|r4||g}|r4|d}WntyT}zt d|dt |WYd}~nd}~ww|duru|duru| ru| } | dd } | | vrud}|dur||gng} t| } | | fS) z       rz/v2/model/infozv2 - returns models available to the user based on their API key permissions. Shows model info from config.yaml (except api key and api base). Filter to just user-added models with ?user_models_only=true)rrrr/z!Specify the model name (optional)z%Only return models added by this userz.Return all models across all teams user is in.riz Page number)rge2z Page sizez3Search model names (case-insensitive partial match)z,Search for a specific model by its unique IDzaFilter models by team ID. Returns models with direct_access=True or teamId in access_via_team_idszLField to sort by. Options: model_name, created_at, updated_at, costs, statuszSort order. Options: asc, descuser_models_onlyinclude_team_modelsmodelIdteamIdsortBy sortOrderc  stdustjsgd|d|dStdurtddtjjidtIdH|dur8t ||tttdIdH\} } n.t tj} t durG| t g7} durTfdd | D} t | |pYd ||tt| d IdH\} } |rst| t|td IdH} |rt|tt| d IdH} t| D]\}}t||dur|ndtd| |<q| dur| rt| | ttdIdH} t| } |durt| } | r| r| dvrtdd| ddt| | | pdd} td| t| |dIdH} t| } t| ||| |dS)zQ BETA ENDPOINT. Might change unexpectedly. Use `/v1/model/info` for now. Nrrrrr@)rmrrsrNrcsg|] }|dkr|qSrrdrrwrdrer%rrz!model_info_v2..rg)r rrrrNrr)r rsrrN)rrNrsr F)r/rSrs)r r^rNrs)rrrzInvalid sortOrder: z. Must be 'asc' or 'desc'r)r rrall_models: %s)rr)r rrrr)rsrrNrr)rrrrrrrrtrrrrDrrrrrrrxrSrr)rr/rrrSrrrrrrrr rrDrrdrwre model_info_v2h%s.            rz/model/streaming_metricsz1View time to first token for models in spend logs)rrr/r_selected_model_group startTimeendTimecstdurttjjddtjd|ptt dd}|p t}| | k}|r.d}nd}t }tj ||||IdH}i}|dur|D]U} | d} | d } | d } d } |rh| d }|} ||vrgi||<n| d }|} | d } ||vrzi||<t| }d| vrt| }d|vr|dd}||| || |<qG g}tt|ddd}|D]}dt|i}||D]\}}|||<q||q|t|dSdS)Ninternal_errorrrra SELECT api_base, model_group, model, "startTime", request_id, EXTRACT(epoch FROM ("completionStartTime" - "startTime")) AS time_to_first_token FROM "LiteLLM_SpendLogs" WHERE "model_group" = $1 AND "cache_hit" != 'True' AND "completionStartTime" IS NOT NULL AND "completionStartTime" != "endTime" AND DATE("startTime") = DATE($2::timestamp) GROUP BY api_base, model_group, model, request_id ORDER BY time_to_first_token DESC; a SELECT api_base, model_group, model, DATE_TRUNC('day', "startTime")::DATE AS day, AVG(EXTRACT(epoch FROM ("completionStartTime" - "startTime"))) AS time_to_first_token FROM "LiteLLM_SpendLogs" WHERE "startTime" BETWEEN $2::timestamp AND $3::timestamp AND "model_group" = $1 AND "cache_hit" != 'True' AND "completionStartTime" IS NOT NULL AND "completionStartTime" != "endTime" GROUP BY api_base, model_group, model, day ORDER BY time_to_first_token DESC; r0r/time_to_first_tokenrgZ request_idrhttps:///openai/rcS|dSNrrdrrdrdre&z)model_streaming_metrics..rbdaterZ all_api_bases)rNr>r)rrr'rrrrrrr~ query_rawrrrrr`rrr)rrrrZ is_same_day sql_query_all_api_bases db_response_daily_entriesr  _api_baserrZ unique_keyZ _request_id_day_combined_model_namerrr model_keylatencyrdrdremodel_streaming_metrics&sn       rz/model/metricsz>View number of requests & avg latency per model on config.yamlz gpt-4-32krcustomercstdurtdddtjd|ptttd}|pt}|dus'|dkr)d}|dus1|dkr3d}d}t}tj ||||||IdH}i} |dur|D]D} | d } | d } | d } | d }| | vrhi| | <t | }| durxd | vrxt | }|durd|vr| dd}| ||| | |<qN g}tt| ddd} | D]}dt |i}| |D]\}}|||<q||q|t|dSdS)N Prisma Client is not initializedrrrr undefinednulla SELECT api_base, model_group, model, DATE_TRUNC('day', "startTime")::DATE AS day, AVG(EXTRACT(epoch FROM ("endTime" - "startTime")) / NULLIF("completion_tokens", 0)) AS avg_latency_per_token FROM "LiteLLM_SpendLogs" WHERE "startTime" >= $2::timestamp AND "startTime" <= $3::timestamp AND "model_group" = $1 AND "cache_hit" != 'True' AND ( CASE WHEN $4 != 'null' THEN "api_key" = $4 ELSE TRUE END ) AND ( CASE WHEN $5 != 'null' THEN "end_user" = $5 ELSE TRUE END ) GROUP BY api_base, model_group, model, day HAVING SUM(completion_tokens) > 0 ORDER BY avg_latency_per_token DESC; r0r/rZavg_latency_per_tokenrrrcSrrrdrrdrdrer&rzmodel_metrics..rbrr)rNr>r'rrrrrrr~rrrrrr`rrr)rrrrrrrr r r r r rr Z_avg_latency_per_tokenrrrrrrrdrdre model_metrics&s^ "       rz/model/metrics/slow_responsesz/View number of hanging requests per model_groupc stdurtdddtjd|dus|dkrd}|dus |dkr"d}|p,tttd}|p2t}tj j p8t }t |}d}tj |||||||IdH}|durm|D]} | d p\d } d | vrh| d d } | | d <qT|S) Nrrrrrrra SELECT api_base, COUNT(*) AS total_count, SUM(CASE WHEN ("endTime" - "startTime") >= (INTERVAL '1 SECOND' * CAST($1 AS INTEGER)) THEN 1 ELSE 0 END) AS slow_count FROM "LiteLLM_SpendLogs" WHERE "model_group" = $2 AND "cache_hit" != 'True' AND "startTime" >= $3::timestamp AND "startTime" <= $4::timestamp AND ( CASE WHEN $5 != 'null' THEN "api_key" = $5 ELSE TRUE END ) AND ( CASE WHEN $6 != 'null' THEN "end_user" = $6 ELSE TRUE END ) GROUP BY api_base ORDER BY slow_count DESC; r0rgrr)rNr>r'rrrrrrqrr`r rr~rrr) rrrrrrr`rr rowr rdrdremodel_metrics_slow_responses 'sF !   rz/model/metrics/exceptionsz7View number of failed requests per model on config.yamlcstdurtdddtjd|ptttd}|pt}|dus'|dkr)d} d}tj |||||IdH}g}t } |durw|D]1} | d d } | d d } | d i} | | d}| | | || D] \}}| |qlqE|t| dS)Nrrrrrrra? WITH cte AS ( SELECT CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base, exception_type, COUNT(*) AS num_rate_limit_exceptions FROM "LiteLLM_ErrorLogs" WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp AND model_group = $3 GROUP BY combined_model_api_base, exception_type ) SELECT combined_model_api_base, COUNT(*) AS total_exceptions, json_object_agg(exception_type, num_rate_limit_exceptions) AS exception_counts FROM cte GROUP BY combined_model_api_base ORDER BY total_exceptions DESC LIMIT 200; Zcombined_model_api_basergtotal_exceptionsrexception_counts)r/r)rexception_types)rNr>r'rrrrrr~rrrr=rrrr)rrrrrrrr rrr r/rrZcurr_rowrrrdrdremodel_metrics_exceptionsc'sD          rcCs|di}t|d}|ikr/|di}|dd}ztj|d}Wn ty.i}Ynw|ikre|di}|dd}|d}t|dkrN|d}z tj||dd}Wn tydi}Ynw|D] \}}||vru|||<qi||d<t|d hd }|S) Nrrwrr/rrrrrZdeployment_dictr) rrrTrr\rrrr)r/rrrr?rrrrdrdre_get_proxy_model_info's@          rz /model/infoz/v1/model/infolitellm_model_idc stdur5z tttjtd}Wn tyi}Ynwtdttd|d}|}t |dhd}d|iSt durAt dd d id t durMt dd d id |durut j |d }|durgt dd d|did t|jddd}d|giSg}tt}t durg}nt }t }t|||d} t|j||d} t| | |ttddt d} t| dkrg} | D]} t j| d}|dur| |qt durt| }ng}|D]}t|d}qtd|d|iS)a Provides more info about each model in /models, including config.yaml descriptions (except api key and api base) Parameters: litellm_model_id: Optional[str] = None (this is the value of `x-litellm-model-id` returned in response headers) - When litellm_model_id is passed, it will return the info for that specific model - When litellm_model_id is not passed, it will return the info for all models Returns: Returns a dictionary containing information about each model. Example Response: ```json { "data": [ { "model_name": "fake-openai-endpoint", "litellm_params": { "api_base": "https://exampleopenaiendpoint-production.up.railway.app/", "model": "openai/fake" }, "model_info": { "id": "112f74fab24a7a5245d2ced3536dd8f5f9192c57ee6e332af0f0512e08bed5af", "db_model": false } } ] } ``` Nrwrr~rrrrrzLLM Model List not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configsr@zLLM Router is not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configs)rmrz Model id = z not found on litellm proxyTr)rrr)rrrrF)rrrrtrrsrrr) rtrrrTrr\rrrrrrrsZget_deploymentrrsrrrrrrrrurrrMrrrrxrS)rrrZ_deployment_infoZ_deployment_info_dictZdeployment_infor rrrrall_models_strZ_relevant_modelsr/Z router_modelsZin_place_modelrdrdre model_info_v1's0            r!r rxc Csg}g}|D] }||vr||q|D]-}|dur||krq|j|d}|dur6|tdi|qt|gd}||qtjdurU|D] }|jtjvrTd|_qI|S)N)rx)rx providersTrd)rZget_model_group_inforrrTZpublic_model_groupsrxZis_public_model_group) rsr rx model_groupsrr/Z_model_group_infomodel_group_infoZmgrdrdre_get_model_group_infov(s.     r%z/model_group/infoc svtdus tdus tsdgiSddlm}||tttttddddtd IdH}t t||d}t ||dIdH}d|iS) a Get information about all the deployments on litellm proxy, including config.yaml descriptions (except api key and api base) - /model_group/info returns all model groups. End users of proxy should use /model_group/info since those models will be used for /chat/completions, /embeddings, etc. - /model_group/info?model_group=rerank-english-v3.0 returns all model groups for a specific model group (`model_name` in config.yaml) Example Request (All Models): ```shell curl -X 'GET' 'http://localhost:4000/model_group/info' -H 'accept: application/json' -H 'x-api-key: sk-1234' ``` Example Request (Specific Model Group): ```shell curl -X 'GET' 'http://localhost:4000/model_group/info?model_group=rerank-english-v3.0' -H 'accept: application/json' -H 'Authorization: Bearer sk-1234' ``` Example Request (Specific Wildcard Model Group): (e.g. `model_name: openai/*` on config.yaml) ```shell curl -X 'GET' 'http://localhost:4000/model_group/info?model_group=openai/tts-1' -H 'accept: application/json' -H 'Authorization: Bearersk-1234' ``` Learn how to use and set wildcard models [here](https://docs.litellm.ai/docs/wildcard_routing) Example Response: ```json { "data": [ { "model_group": "rerank-english-v3.0", "providers": [ "cohere" ], "max_input_tokens": null, "max_output_tokens": null, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": null, "tpm": null, "rpm": null, "supports_parallel_function_calling": false, "supports_vision": false, "supports_function_calling": false, "supported_openai_params": [ "stream", "temperature", "max_tokens", "logit_bias", "top_p", "frequency_penalty", "presence_penalty", "stop", "n", "extra_headers" ] }, { "model_group": "gpt-3.5-turbo", "providers": [ "openai" ], "max_input_tokens": 16385.0, "max_output_tokens": 4096.0, "input_cost_per_token": 1.5e-06, "output_cost_per_token": 2e-06, "mode": "chat", "tpm": null, "rpm": null, "supports_parallel_function_calling": false, "supports_vision": false, "supports_function_calling": true, "supported_openai_params": [ "frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "response_format" ] }, { "model_group": "llava-hf", "providers": [ "openai" ], "max_input_tokens": null, "max_output_tokens": null, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": null, "tpm": null, "rpm": null, "supports_parallel_function_calling": false, "supports_vision": true, "supports_function_calling": false, "supported_openai_params": [ "frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "response_format" ] } ] } ``` Nrr)rFr)rsr rx)r#r) rrsrrrurtrNrqrrr%r)rrxrr r#rdrdrer$(s4#    r$z/model/settingszMReturns provider name, description, and required parameters for each providercs0g}tjD]}|t|tj|ddq|S)a? Used by UI to generate 'model add' page { field_name=field_name, field_type=allowed_args[field_name]["type"], # string/int field_description=field_info.description or "", # human-friendly description field_value=general_settings.get(field_name, None), # example value } r)rfields)rTZ provider_listrZ ProviderInfoZget_provider_fields)Z returned_listrrdrdremodel_settingsZ)s  r'z/alerting/settingszFReturn the configurable alerting param, description, and current valuer_c s tdurtddtjjid|jtjkr%tdddtj j|jidtj j j ddidIdH}|durL|j durLt|j }|di}|d }ni}d}d d id d id d id d id d id d id d id d id d id }tj}|j}g}d} td rttd trdtd vrd} td|dd d| |durdndddd} || tjD]5\} } | |vrd} | |vrd} nd} t| || d | jpd|| d| | j| dkrdndd} || q|S)Nrrr@{}, your role={}rrurrcr_rUBooleanInteger) slack_alertingZdaily_report_frequencyZreport_check_intervalZbudget_alert_ttlZoutage_alert_ttlregion_outage_alert_ttlZminor_outage_alert_thresholdZmajor_outage_alert_thresholdZmax_outage_alert_list_sizeFreTr+zhEnable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.) field_name field_typefield_description field_value stored_in_dbfield_default_valueZ premium_fieldrgr,)rNrr)rrrr;rrqnot_allowed_accessr~rrrrrrqrrcrrurrr.rr model_fieldsrrrg)rrdb_general_settings_dictZalerting_args_dictZalerting_values allowed_argsZ_slack_alertingZ_slack_alerting_args_dict return_valZis_slack_enabled _response_objr- field_info _stored_in_dbrdrdrealerting_settings{)s                 r;z/queue/chat/completionsr})rrr/c s i}z|IdH}t|j|jt|t|d|d<td|t ddp5t p5|p5| dd|d<| dddurJ|j durJ|j |d<d|vrRi|d<|j |dd<|j |dd <t|}|d d||dd <t|d d|dd <|j |dd<t|dd|dd<t|dd|dd<t|dd|dd<t|j|dd<trt|d<trt|d<trt|d<trt|d<tdurtddtjjidtjd-i|IdH}d|vr|ddurtt|||dd d!WS|jd"t|d#i|WStyS}zGtj |||d$IdHt!|tr5t"t|d%d&t|d't#j$t|d(d)t|d*t%j&d+t!|t"r=|t"d,t|t#j$t|d(d)t%j&d+d}~ww).N)rrVrrQZproxy_server_requestzreceiving data: %sZcompletion_modelr/rrZ user_api_keyZuser_api_key_metadata authorizationrrfZuser_api_key_aliasrr^rZobject_permission_idZ!user_api_key_object_permission_idZteam_object_permission_idZ&user_api_key_team_object_permission_idrrorprnr0rrr@rTrr"r$zx-litellm-prioritypriorityrrAAuthentication Error(r@rrrrAuthentication Error, rd)'rrrrVrrrxrSrurrtr\rrrrr{r|rrzrsrr)rzrZschedule_acompletionr1rrr=r\rqrrr>r= auth_errorr'r-)rrr/rr_headersrrerdrdreasync_queue_request)s                    rC)rr/csddlm}tt|j}td}|dr|d7}n|d7}|dur4ddlm }|t d d d d Sddlm }|t d d d d S) z Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env PROXY_BASE_URL should be the your deployed proxy endpoint, e.g. PROXY_BASE_URL="https://litellm-production-7002.up.railway.app/" Example: r)ui_linkZ UI_USERNAMErz sso/callbackz /sso/callbackN) HTMLResponseF)Zshow_deprecation_bannerrrr) Zlitellm.proxy.proxy_serverrDrrbase_urlrrSr!fastapi.responsesrEr)rrD redirect_urlZ ui_usernamerErdrdrefallback_loginc*s         rJz/loginc sddlm}m}ddlm}|IdH}t|d}t|d}|||tt dIdH}||t t d}ddl } | j tt|tttdd } |t|j} | d rY| d 7} n| d 7} | d 7} t| dd} | jd| d| S)Nrauthenticate_usercreate_ui_token_objectrusernamerrOrrCrN login_resultrur9HS256 algorithmrui//ui/?login=successi/)rrr[rcr)litellm.proxy.auth.login_utilsrLrMrrformrrrCrNrur9jwtencoderrrGr!r0 set_cookie) rrLrMrr[rOrrRreturned_ui_token_objectr\ jwt_tokenlitellm_dashboard_uiZredirect_responserdrdrelogin*s>     rbz /v2/loginc s|ddlm}m}ddlm}ze|IdH}t|d}t|d}|||tt dIdH}||t t d}ddl } | j tt|tttdd } |t|j} | d rZ| d 7} n| d 7} | d 7} td| itjd} | jd| d| WSty} z=tdt| t| tr| t| trtt| dt| tjt| ddt| dtj dt| }t|tjdtj dd} ~ ww)NrrKrNrOrrPrQrSrTrrVrWrXrIrFr[rYz>litellm.proxy.proxy_server.login_v2(): Exception occurred - {}rArrrr)!rZrLrMrrrrrrCrNrur9r\r]rrrGr!r.r'Z HTTP_200_OKr^r\rxrUrqrr>rrr=rAr)rrLrMrrQrOrrRr_r\r`raZ json_responsererrdrdrelogin_v2*sv         rcz/onboarding/get_token invite_linkc sddlm}tdurtdtjdtjdtdur#t ddt j j id tj jjd |id IdH}|durr=rAr'rrNrr)rrr~litellm_invitationlinkr rTr)get_utc_datetimer expires_at is_acceptedrr\rlrrr$Zmax_ui_session_budgetrrrGr!r\rr9rurrr]rrrq)rdrre invite_obj utc_now_dateexpires_at_dateuser_objrlrrcrar\rmr_r`rdrdre onboarding*s          rvz/onboarding/claim_tokenrcsBtdurtddtjjidtjjjd|jidIdH}|dur)tdddidt j }|j }||krAtddd id|jd urNtddd id|j|jkrbtddd |j|jidt|jd }tjjjd|jid|idIdH}|durtdddidt j }tjjjd|ji||d |jddIdH|S)a& Special route. Allows UI link share user to update their password. - Get the invite link - Validate it's still 'valid' - Check if user within initial session (prevents abuse) - Get user from db - Update user password This route can only update user password. Nrrr@rbrrfrgrhTrizwInvalid invitation link. The user id submitted does not match the user id this link is attached to. Got={}, Expected={}rar\rr%rj) accepted_atrprqrr)rNrr)rrr~rnr Zinvitation_linkrTr)rorrprqr\rqrrrr=)rrrrsrtZ hash_passwordrur0rdrdreclaim_onboarding_linkq+sd          rxz /get_logo_urlcCstdd}d|iS)z)Get the current logo URL from environment UI_LOGO_PATHrgZlogo_url)rrS) logo_pathrdrdre get_logo_url+s r{z /get_imagec stjtjt}tj|d}tdddk}|r dn|}td|}tj|sbztj |ddt d |Wn"t t fya}zt d |d |d |}WYd }~nd }~ww||krmtj|dn|}||kr{tj|s{|}t|tjr|n|}tj|d}td|} t d| | |kr| dstj| rt| ddSt d| d|} tj|rt|ddS| dr=zMddlm} ddlm} | | jddid} | | Id H} | jdkrt|d}|| jWd n 1swYt|ddWSt|ddWSty<}zt d| d |t|ddWYd }~Sd }~wwt| ddS)zGet logo to show on admin UIzlogo.jpgrrgrz/var/lib/litellm/assetsZLITELLM_ASSETS_PATHTr zCreated assets directory at z"Cannot create assets directory at rZz@. Logo caching may not work. Using current directory for assets.Nzcached_logo.jpgryzReading logo from path: %szhttp://rz image/jpegr=zUI_LOGO_PATH 'z.' does not exist, falling back to default logorget_async_httpx_clienthttpxSpecialProvidertimeout@Z llm_providerrlrwbzError downloading logo from )rrWrr__file__rrSrrrrxrSrrrcaccessW_OKrr-&litellm.llms.custom_httpx.http_handlerr~litellm.types.llms.custom_httprUIrrrr]rr\)r+Zdefault_site_logo is_non_rootZdefault_assets_dirZ assets_dirreZ default_logo cache_dir cache_pathrzr~r async_clientrfrdrdre get_image+sr                 rz /get_faviconc sddlm}tjtjt}tj|ddd}tdd}|s4tj |r.t |dd St d d d | d rzLddl m}ddlm}||jddid}||IdH}|jdkrk|jdd}||j|dWStd||jtj |rt |dd WSt d dd t yty} ztd|| tj |rt |dd WYd} ~ St d dd d} ~ wwtj |rt |dd Stj |rt |dd St d dd )z$Get custom favicon for the admin UI.rr"rrz favicon.icoZLITELLM_FAVICON_URLrgz image/x-iconr=r zDefault favicon not foundr@r|r}rrrrNr content-type)rr$z*Failed to fetch favicon from %s: status %szFavicon not foundz%Error downloading favicon from %s: %s)rHr"rrWrrrrrSrr-rrrr~rrrrrrrrxrcr\rS) r"r+Zdefault_faviconZ favicon_urlr~rrr content_typererdrdre get_favicon,s`                   rz/invitation/newz Invite Links)rrrr/c sz_ddlm}tdurtddtjjid|jtj kp&t |tt t dIdH}|s8tddd tjj|jid|jtj krUt|j|tt t d IdH}|sUtddd id|||d IdH}|WStyp}zt|d}~ww) % Allow admin to create invite links, to onboard new users to Admin UI. ``` curl -X POST 'http://localhost:4000/invitation/new' -H 'Content-Type: application/json' -d '{ "user_id": "1234" // 👈 id of user in 'LiteLLM_UserTable' }' ``` r)create_invitation_for_userNrrr@rr()Ztarget_user_idrrNrrrqzGYou can only create invitations for users in your organization or team.)rr)Z0litellm.proxy.management_helpers.user_invitationrrNrr)rrrr;rrrrrqrqr3rr\r\r)rrrZ has_accessZ can_inviterrerdrdrenew_invitation[,s`        rz/invitation/info invitation_idcs~tdurtddtjjid|jtjkr$tdddtj j|jidtj j j d|idIdH}|dur=tdddid|S) rNrrr@r(rbr-Invitation id does not exist in the database.) rNrr)rrrr;rrqr3r~rnr )rrrrdrdreinvitation_info,s0    rz/invitation/updatecstdurtddtjjid|jdur tddd|jidtj }tj j j d|j i|j |j|||jdd IdH}|durItddd id|S) uV Update when invitation is accepted ``` curl -X POST 'http://localhost:4000/invitation/update' -H 'Content-Type: application/json' -d '{ "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable' "is_accepted": True // when invitation is accepted }' ``` Nrrr@rz'Unable to identify user id. Received={}rb)rbrqrwrprrr%r)rNrr)rrr\rqrTr)ror~rnr=rrq)rrr0rrdrdreinvitation_update,s<     rz/invitation/deletecstdurtddtjjid|jtjk}t|tt t dIdH}|s3|s3tddd tj j|jid|r_|s_tj jjd|jidIdH}|durQtddd id|j|jkr_td dd idtj jjd|jidIdH}|durytddd id|S) u  Delete invitation link ``` curl -X POST 'http://localhost:4000/invitation/delete' -H 'Content-Type: application/json' -d '{ "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable' }' ``` Nrrr@rr(rbrrz=Organization admins can only delete invitations they created.)rNrr)rrrr;rrrrrqrqr3r~rnr rrqr\delete)rrZis_proxy_adminZis_other_adminZ invitationrrdrdreinvitation_delete-s\       rz/config/updatez config.yaml config_infoc s,z9ddl} tdurtdtdurtdddid|jdd }t|}|D]\}}|d krItjj j d |i||d d |iddIdHq+t IdH}t d||jdur|di|jjdd }|d}|D]*\}}|dkrd|vrddgi}nt|dtrd|dvr|dd|||<qq||d<|jdur|di|j}|D] \}}t|d} | ||<q|d} |D] \}}||| |<q|jdur$|di|j} i| |d|d<d| vr$d|dvr$t|ddtr$t| dtr$t| d} |dd| } tt| } | |dd<t j|dIdHt jttdIdHddiWSty}zMt dt|t t !t|trwt"t#|ddt|d t$j%t#|d!d"t#|d#t&j'd$t|t"r|t"d%t|t$j%t#|d!d"t&j'd$d}~ww)&z For Admin UI - allows admin to update config via UI Currently supports modifying General Settings + LiteLLM settings rNzNo DB ConnectedTrrCSet `'STORE_MODEL_IN_DB='True'` in your env to enable this feature.r@rrrr"rr#r%zLoaded config: %srurbr_rerrrvrYrrr^zConfig updated successfullyzBlitellm.proxy.proxy_server.update_config(): Exception occured - {}rAr?r@rrrrr@)(rrNr\r=rrZjsonify_objectrr~rr/rrrxrSrurrrrrrrrvrFrrrrqrrqrrtrur>rr=rAr'r-)rrZupdated_settingsrrrZupdated_general_settingsZ_existing_settingsZ_updated_environment_variablesrZ_existing_env_variablesZupdated_litellm_settingsZ$updated_success_callbacks_normalizedZcombined_success_callbackrerdrdre update_confign-s                            rz/config/field/updatec s. tdurtddtjjid|jtjkr tddtjjid|j t j vr2tddd |j idz t di|j |j iWntyTtddd t|j idwtjjjddid IdH}|dusk|jdurni}nt|j}|j ||j <tjjjddidt|d d t|id d IdH}|S)z= Update a specific field in litellm general settings Nrrr@Invalid field={} passed in.z)Invalid type of field value={} passed in.rrurr"rr#r%rd)rNrr)rrrr;rr3r-r-r4rqr0r\rUr~rrrrr/rrgrrrrurrdrdreupdate_config_general_settings .sT           rz/config/field/infor-cs tdurtddtjjid|jtjkr tddtjjid|t j vr0tddd |idtj j jddidIdH}|dusF|jdurQtddd |idt|j}||vrbt|||d Stddd |id) Nrrr@rrrurzField name={} not in DB)r-r0)rNrr)rrrr;rr3r-r4rqr~rrrrr+)r-rrrurdrdreget_config_general_settingsW.sD         rz /config/listrc sT tdurtddtjjid|jtjkr%tdddtj j|jidtj j j ddidIdH}|durA|j durAt|j }ni}d d id d id d id d id d id d id d id d id did did d id d id }g}tjD]\}}||vr'||d }|d kr|dkrtg} ng} | D]W} dd| jD} d} | jD]\} }t|dr|jdur|j| | _| d7} qd}||vrd}n|tvrd}t|||d |jpdt|d||j| d}||qqqd} d}||vrd}n|tvrd}t|d}|dur||vr||}t|||d |jpd|||j| d}||qq|S)z List the available fields + current values for a given type of setting (currently just 'general_settings'user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),) Nrrr@r(rrurrUr*Z PydanticModelr)Stringr ) rrZmax_request_size_mbZmax_response_size_mbr)r=rrZmcp_internal_ip_rangesZmcp_trusted_proxy_rangesZalways_include_stream_usageZ!forward_client_headers_to_llm_apir)c Ss,g|]\}}t||jdt|dddqS)rgN)r-r.r/r2r1)r1r\rur)r sub_fieldZsub_field_typerdrdrer.s z#get_config_list..rrriTFrg)r-r.r/r0r1r2 nested_fields)rNrr)rrrr;rrqr3r~rrrrr-r4rr<__annotations__rrr/rur.rrgr)rrrr5r6r7r-r9Ztyped_dict_typeZpydantic_class_listZpydantic_classrrOrZsub_field_infor:r8Z _field_valuerdrdreget_config_list.s             -    rz/config/field/deletecs tdurtddtjjid|jtjkr%tdddtj j|jid|j t j vr7tddd|j idtj jjddid IdH}|dusM|jdurYtddd |j idt|j}||j dtj jjddidt|d d t|id dIdH}|S)z| Delete the db value of this field in litellm general settings. Resets it to it's initial default value on litellm. Nrrr@r(rrrurzField name={} not in configr"rr#r%)rNrr)rrrr;rrqr3r-r-r4r~rrrrrr/rrgrrdrdredelete_config_general_settings)/sL       rz/config/callback/deletec sttdurtddtjjid|jtjkr$tdddtj j|jidt dur0tdddidzWt IdH}|j }|d i}|d g}||vrYtd dd |d id||||d id <t j|dIdHt jttdIdHd|||tdWStyty}ztdt|tttdt|t j!dt"j#dd}~ww)z> Delete specific logging callback from configuration. Nrrr@r(TrrrvrYr z Callback 'z#' not found in active configurationrrzSuccessfully deleted callback: )r^Zremoved_callbackZremaining_callbacksZ deleted_atzClitellm.proxy.proxy_server.delete_callback(): Exception occurred - zError deleting callback: callback_namer)$rNrr)rrrr;rrqr3r=rrrrrremoverrrrqrrr.r\rxrrrSrtrur>r=internal_server_errorr'r)rrrrrvrrerdrdredelete_callbackq/sz              rz/get/config/callbacks)rr/rc szddl}t}tIdH}|di}|di}|di}|dg}|dg}|dg}d d } | |}| |}| |}g} |D] } | t| d |qI|D] } | t| d |qW|D] } | t| d |qe|dg} g} d| vrdg}i}|D]!}||d}|durtdd}|||<qt ||d}|||<qt j j }t j }t j j}| d|||dgd}i}|D]}||d}|durd||<qt ||d}|||<q| d|dtduri}nt}d | | ||dWStyO}zFtdt|t|tr1tt|ddt|dtjt|ddt|dtjdt|tr9|tdt|tjt|ddtjdd}~ww) z~ For Admin UI - allows admin to view config via UI # return the callbacks and the env variables for the callback rNrvrurrYrr cSs t|tr|gS|durgS|SrB)rr)rNrdrdrenormalize_callback/s z&get_config..normalize_callbackrrZsuccess_and_failurer_reZSLACK_WEBHOOK_URLr[)r variablesZ active_alertsZalerts_to_webhook)Z SMTP_HOST SMTP_PORTZ SMTP_USERNAMEZ SMTP_PASSWORDZSMTP_SENDER_EMAILZTEST_EMAIL_ADDRESSZEMAIL_LOGO_URLZEMAIL_SUPPORT_CONTACTemail)rr)r'r ZalertsrZavailable_callbacksz?litellm.proxy.proxy_server.get_config(): Exception occured - {}rAr?r@rrrrr@)rZ AllCallbacksrrrrrGrrSrrqrraZ_all_possible_alert_typesrbrsZ get_settingsr\rxrUrqrrrr>rr=rAr'r-)rZall_available_callbacksrrrrZ_success_callbacksZ_failure_callbacksZ_success_and_failure_callbacksrZ_data_to_return _callbackZ _alertingZ alerting_dataZ _slack_varsZ_slack_env_varsZ_varZ env_variablerZ_decrypted_valueZ_alerting_typesZ_all_alert_typesZ_alerts_to_webhookZ _email_varsZ_email_env_varsZ_router_settingsrerdrdrer/s                          rz /config/yamlcs ddiS)a This is a mock endpoint, to show what you can set in config.yaml details in the Swagger UI. Parameters: The config.yaml object has the following attributes: - **model_list**: *Optional[List[ModelParams]]* - A list of supported models on the server, along with model-specific configurations. ModelParams includes "model_name" (name of the model), "litellm_params" (litellm-specific parameters for the model), and "model_info" (additional info about the model such as id, mode, cost per token, etc). - **litellm_settings**: *Optional[dict]*: Settings for the litellm module. You can specify multiple properties like "drop_params", "set_verbose", "api_base", "cache". - **general_settings**: *Optional[ConfigGeneralSettings]*: General settings for the server like "completion_model" (default model for chat completion calls), "use_azure_key_vault" (option to load keys from azure key vault), "master_key" (key required for all calls to proxy), and others. Please, refer to each class's description for a better understanding of the specific attributes within them. Note: This is a mock endpoint primarily meant for demonstration purposes, and does not actually provide or change any configurations. ZhelloZworldrd)rrdrdreconfig_yaml_endpointz0srz/reload/model_cost_mapc sn|jtjkrtdd|jdztdurtdddddlm}tj}||d }|t_ t tj |d t }|atjjjd d id IdH}d}|rX|jrX|jd}tjjjd d id tdddddt|ddiddIdH|r~t|nd}td|d|dd||dWSty}ztdt|tddt|dd}~ww)z ADMIN ONLY / MASTER KEY Only Endpoint Manually reload the model cost map from the remote source. This will fetch fresh pricing data from the model_prices_and_context_window.json file. r2Access denied. Admin role required. Current role: r@Nr!Database connection not availablerrr0r rrrrTr!r"rr#r%zCModel cost map reloaded successfully in current pod. Models count: z"Price data reloaded successfully! z models updated.r)r^r' models_count timestampz!Failed to reload model cost map: )rr;rrrNr*rrTr+r,rNr-rr&r.r'r~rr rrr/r&rrxrRr\rUr) rrr+r5r0Zexisting_configZexisting_intervalrrerdrdrereload_model_cost_map0sf           rz/schedule/model_cost_map_reloadhoursc |jtjkrtdd|jd|dkrtdddzCtdur&tdd dtjjjd d id t|d d ddt|d d iddIdHt d|dd|dd|t dWSty}zt dt|tddt|dd}~ww)z ADMIN ONLY / MASTER KEY Only Endpoint Schedule periodic reload of the model cost map. This will create a background job that reloads the model cost map every specified hours. rrr@rrHours must be greater than 0NrrrrFr!r"rr#r%z*Model cost map reload scheduled for every  hoursrr^r'rrz*Failed to schedule model cost map reload: rr;rrrNr~rr/r&rxrRrr&r.r\rUrrrrerdrdreschedule_model_cost_map_reload0X         rc |jtjkrtdd|jdz'tdurtdddtjjjddid IdHt d d d t d WSt yY}ztd t|tdd t|dd}~ww)zp ADMIN ONLY / MASTER KEY Only Endpoint Cancel the scheduled periodic reload of the model cost map. rrr@Nrrrrrz(Model cost map reload schedule cancelledrr^r'rz(Failed to cancel model cost map reload: rr;rrrNr~rrrxrRrr&r.r\rUrrrerdrdrecancel_model_cost_map_reload01s8     rz&/schedule/model_cost_map_reload/statusc |jtjkrtdd|jdztdttdur+tddddddWStjj j d d id IdH}|dusA|j durNtd dddddWS|j }| d }|durgtddddddWSt }d}tdurzt t}||}|d}||kr|t|d}Wnty} ztd| WYd} ~ nd} ~ wwd|t|dWSty} ztdt| tddt| dd} ~ ww)zo ADMIN ONLY / MASTER KEY Only Endpoint Get the status of the scheduled model cost map reload job. rrr@z4Checking model cost map reload status. Last reload: N/No database connection, returning not scheduledFZ scheduledrZlast_runnext_runrrrz,No model cost map reload configuration foundr/No interval configured, returning not scheduledr8rrTz,Failed to get model cost map reload status: r)rr;rrrxrRr'rNr~rr rrrr&r(r)rr.r\rcrUr rr rrr0rr2r3r4rerdrdre get_model_cost_map_reload_statusa1s            rz/model/cost_map/sourcec s|jtjkrtdd|jdzddlm}|}tjr#ttjnd}i|d|iWSt yN}zt dt |tddt |dd }~ww) a ADMIN ONLY / MASTER KEY Only Endpoint Returns information about where the current model cost/pricing data was loaded from. Response fields: - source: "local" (bundled backup) or "remote" (fetched from URL) - url: the remote URL that was attempted (null when env-forced local) - is_env_forced: true if LITELLM_LOCAL_MODEL_COST_MAP=True forced local usage - fallback_reason: human-readable reason why remote failed (null on success) - model_count: number of models in the currently loaded cost map rrr@r)get_model_cost_map_source_info model_countz*Failed to get model cost map source info: rN) rr;rrr*rrTr,rr\rxrUr)rrZ source_inforrerdrdreget_model_cost_map_source1s2     rz/reload/anthropic_beta_headersc sR|jtjkrtdd|jdzvtdurtdddddlm}|}t}| a tj j j d d id IdH}d}|rG|jrG|jd }tj j jd d id tdd dddt|d diddIdHtdd|D}td|d|dd|| dWSty}ztdt|tddt|dd}~ww)z ADMIN ONLY / MASTER KEY Only Endpoint Manually reload the Anthropic beta headers configuration from the remote source. This will fetch fresh configuration from the anthropic_beta_headers_config.json file. rrr@Nrrrr7rr6rrTr!r"rr#r%css|] }|dvrdVqdS))r9rriNrdr:rdrdrer12sz0reload_anthropic_beta_headers..zOAnthropic beta headers config reloaded successfully in current pod. Providers: zrerdrdrereload_anthropic_beta_headers1sf          rz'/schedule/anthropic_beta_headers_reloadc r)z ADMIN ONLY / MASTER KEY Only Endpoint Schedule periodic reload of the Anthropic beta headers configuration. This will create a background job that reloads the configuration every specified hours. rrr@rrrNrrrr6Fr!r"rr#r%z2Anthropic beta headers reload scheduled for every rrrz2Failed to schedule anthropic beta headers reload: rrrdrdre&schedule_anthropic_beta_headers_reloadG2rrc r)z ADMIN ONLY / MASTER KEY Only Endpoint Cancel the scheduled periodic reload of the Anthropic beta headers configuration. rrr@Nrrrr6rz0Anthropic beta headers reload schedule cancelledrrz0Failed to cancel anthropic beta headers reload: rrrdrdre$cancel_anthropic_beta_headers_reload2s:      rz./schedule/anthropic_beta_headers_reload/statusc r)zw ADMIN ONLY / MASTER KEY Only Endpoint Get the status of the scheduled Anthropic beta headers reload job. rrr@z.custom_send)receivesendrcSsi|] \}}||qSrd)rB)rrrrdrdrer3r]z%dynamic_mcp_route..rzapplication/json)rrrr$z%Error handling dynamic MCP route for rZrzInternal server error: )rWrRZ#litellm.proxy.auth.ip_address_utilsrZlitellm.types.mcprZget_mcp_client_ipZget_mcp_server_by_namerrr-litellm.proxy._experimental.mcp_server.serverrrZstarlette.responsesr"rr\rxrr) rrrRrrrZ mcp_serverrrrr"Z headers_dictrerdrredynamic_mcp_route3sT          r)rWr)NNrdrB)NNNNFFNNr?NFTTNFFN)FN)r(;rrenumrrrarrmZsecretsrrrrrBrtwarningsrrrtypingrrrrr r r r r rrrrrrrwZwebsockets.exceptionsZpydanticrrZ litellm._uuidrrrrrrrrrrrrr r!r"r#r$Z*litellm.litellm_core_utils.litellm_loggingr%Z*litellm.litellm_core_utils.safe_json_dumpsr&Zlitellm.proxy._typesr'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrEZ)litellm.proxy.common_utils.callback_utilsrFrGZ)litellm.proxy.common_utils.realtime_utilsrIr?rJrKrLrMrZrNrOrarPZopentelemetry.tracerQ_SpanZ"litellm.integrations.opentelemetryrRrXrrrffilterwarnings UserWarningrhrrWinsertrrvbackoffZfastapirRrZapscheduler.schedulers.asynciorirrerorr collectionsrs contextlibrt functoolsrurTrvrwrxryr<rzr{Z#litellm.caching.redis_cluster_cacher|r}r~rrrrrrrrrrrrZlitellm.exceptionsrZ%litellm.integrations.custom_guardrailrZ"litellm.integrations.custom_loggerrZ1litellm.integrations.SlackAlerting.slack_alertingrZ'litellm.litellm_core_utils.core_helpersrrZ.litellm.litellm_core_utils.credential_accessorrrZLiteLLMLoggingObjZ0litellm.litellm_core_utils.sensitive_data_maskerrrrrZ&litellm.llms.vertex_ai.vertex_llm_baserZ;litellm.proxy._experimental.mcp_server.byok_oauth_endpointsrZmcp_byok_oauth_routerZ=litellm.proxy._experimental.mcp_server.discoverable_endpointsZ!mcp_discoverable_endpoints_routerZ5litellm.proxy._experimental.mcp_server.rest_endpointsZmcp_rest_endpoints_routerrrZmcp_appZ4litellm.proxy._experimental.mcp_server.tool_registryrZ+litellm.proxy.agent_endpoints.a2a_endpointsZ a2a_routerrVrZ'litellm.proxy.agent_endpoints.endpointsZagent_endpoints_routerZ0litellm.proxy.agent_endpoints.model_list_helpersrrZ5litellm.proxy.analytics_endpoints.analytics_endpointsZanalytics_routerZ7litellm.proxy.anthropic_endpoints.claude_code_endpointsrZ+litellm.proxy.anthropic_endpoints.endpointsZanthropic_routerZ2litellm.proxy.anthropic_endpoints.skills_endpointsZanthropic_skills_routerZlitellm.proxy.auth.auth_checksrrrZlitellm.proxy.auth.auth_utilsrZlitellm.proxy.auth.handle_jwtrZ"litellm.proxy.auth.litellm_licenserr rrrrrZ$litellm.proxy.auth.user_api_key_authrrrZ)litellm.proxy.batches_endpoints.endpointsZbatches_routerZlitellm.proxy.caching_routesZcaching_routerZ'litellm.proxy.common_request_processingrrrZ&litellm.proxy.common_utils.debug_utilsrZdebugging_endpoints_routerZ0litellm.proxy.common_utils.encrypt_decrypt_utilsrrZ.litellm.proxy.common_utils.html_forms.ui_loginrZ-litellm.proxy.common_utils.http_parsing_utilsrrrrZ,litellm.proxy.common_utils.load_config_utilsrrZ0litellm.proxy.common_utils.openai_endpoint_utilsrZ&litellm.proxy.common_utils.proxy_staterZ+litellm.proxy.common_utils.reset_budget_jobrZ(litellm.proxy.common_utils.swagger_utilsrZ+litellm.proxy.container_endpoints.endpointsZcontainer_routerZ,litellm.proxy.credential_endpoints.endpointsZcredential_routerrrZ"litellm.proxy.db.exception_handlerrZ!litellm.proxy.discovery_endpointsrZ-litellm.proxy.fine_tuning_endpoints.endpointsZfine_tuning_routerrZ(litellm.proxy.google_endpoints.endpointsZ google_routerZ,litellm.proxy.guardrails.guardrail_endpointsZguardrails_routerZ(litellm.proxy.guardrails.init_guardrailsrrZlitellm.proxy.health_checkrrZ health_routerZ,litellm.proxy.hooks.model_max_budget_limiterrZ.litellm.proxy.hooks.prompt_injection_detectionrZ-litellm.proxy.hooks.proxy_track_cost_callbackrZ'litellm.proxy.image_endpoints.endpointsZ image_routerZ$litellm.proxy.litellm_pre_call_utilsrZ9litellm.proxy.management_endpoints.access_group_endpointsZaccess_group_routerZ>litellm.proxy.management_endpoints.budget_management_endpointsZbudget_management_routerrZcache_settings_routerZ@litellm.proxy.management_endpoints.callback_management_endpointsZ$callback_management_endpoints_routerrZconfig_override_routerrrrZ7litellm.proxy.management_endpoints.compliance_endpointsZcompliance_routerZ9litellm.proxy.management_endpoints.cost_tracking_settingsZcost_tracking_settings_routerZ5litellm.proxy.management_endpoints.customer_endpointsZcustomer_routerZ@litellm.proxy.management_endpoints.fallback_management_endpointsZfallback_management_routerZ:litellm.proxy.management_endpoints.internal_user_endpointsZinternal_user_routerrZlitellm.proxy.pass_through_endpoints.llm_passthrough_endpointsrZllm_passthrough_routerrrZrZpass_through_routerZ,litellm.proxy.policy_engine.policy_endpointsZpolicy_crud_routerZ4litellm.proxy.policy_engine.policy_resolve_endpointsZpolicy_resolve_routerrBZprompts_routerZlitellm.proxy.public_endpointsZpublic_endpoints_routerZ%litellm.proxy.rag_endpoints.endpointsZ rag_routerZ(litellm.proxy.rerank_endpoints.endpointsZ rerank_routerZ.litellm.proxy.response_api_endpoints.endpointsZresponse_routerZlitellm.proxy.route_llm_requestrZ(litellm.proxy.search_endpoints.endpointsZ search_routerZ5litellm.proxy.search_endpoints.search_tool_managementZsearch_tool_management_routerrZcloudzero_routerZ7litellm.proxy.spend_tracking.spend_management_endpointsZspend_management_routerZ1litellm.proxy.spend_tracking.spend_tracking_utilsrZlitellm.proxy.types_utils.utilsrrZui_crud_endpoints_routerrrrrrrrrrrrrrrrrrZ.litellm.proxy.vector_store_endpoints.endpointsZvector_store_routerZ9litellm.proxy.vector_store_endpoints.management_endpointsZvector_store_management_routerZ4litellm.proxy.vector_store_files_endpoints.endpointsZvector_store_files_routerZ4litellm.proxy.vertex_ai_endpoints.langfuse_endpointsZlangfuse_routerZ'litellm.proxy.video_endpoints.endpointsZ video_routerZlitellm.routerrrrrZlitellm.schedulerrrZ*litellm.secret_managers.aws_secret_managerrZ"litellm.secret_managers.google_kmsrrrrrrZ)litellm.types.integrations.slack_alertingrZlitellm.types.llms.anthropicrrrrZlitellm.types.llms.openairZClitellm.types.proxy.management_endpoints.model_management_endpointsrZ/litellm.types.proxy.management_endpoints.ui_ssorr Zlitellm.types.realtimer r@r r rsr rrZlitellm.types.schedulerrZ"litellm.types.secret_managers.mainrrrrZ ModelMapInforrrZlitellm._versionrr\Zsuppress_debug_inforrrrrrrrr r!r"r#r$r%r&r'Zfastapi.encodersr(Zfastapi.middleware.corsr)Zfastapi.openapi.docsr+Zfastapi.openapi.utilsr,rHr-r.r/r0r1Zfastapi.routingr2Zfastapi.securityr3Zfastapi.security.api_keyr4Zfastapi.staticfilesr5Zlitellm.types.agentsr6Zenterprise_routerZlitellm.proxy.enterpriseproxyZ enterpriseZ*litellm_enterprise.proxy.enterprise_routesZ_enterprise_routerZ%litellm_enterprise.proxy.proxy_serverr7r8rrrr9rZairgapped_license_datar:rSr;Z proxy_staterr=rr>r?rlrDZfallback_login_linkZmodel_hub_linkZ ui_messageZ chat_linkZcustom_swagger_message_titleZ _descriptionrOr]rfrZ#vertex_live_passthrough_vertex_baserrrZopenapiEnumrZexception_handlerrZoriginsrrr+rZpackaged_ui_pathrZlitellm_asset_prefixrrr rkrrrZdefault_runtime_ui_pathZruntime_ui_pathrZshould_use_runtime_pathZis_pre_restructuredZ has_contentrRrcrrr is_writablerrootdirsr#r`r$r!rrr`rrZmodified_contentr]UnicodeDecodeErrorrrr,r&rUZadd_middlewarer-rr.rr1rzrtrurr|r{r~Z user_configryrDZ local_loggingr}rsrrurr2Zlog_filerrCr3rErNrrrrrZmodel_max_budget_limiterrXrYrtr6r7r9rFrGrHrIrJrKrrLrMrr:rrr;ryrzr<r{r|r,r}rIr-rVrr=r>rqZ async_resultZcelery_app_connZ celery_fnrr'r;rXrFrLrNrWrZrrrrrrrrrrrhrrrrrrrrrrrrrpostr6r:r;rEbytesrXr\rcrdrhrmryr|r}rrrrrrrZ litellm.llms.base_llm.base_utilsrrrrrrrrrrrrrrrrrrrrrrrrrr!r%r$r'r;rCrJrbrcrvrxr{rrrrrrrrrrrrrrrrrrrrrrrrrZinclude_routerZ api_routerrdrdrdres@ @ D  !            @                              ""***"   2           [   I    /               (            ( .   D"- w 9 /g   ?     #  9     y   m    m      % z  X  X   W  W   U   Y   U   l9 "   --,E  L -P (y/    $ } l S Q+  " >  g  t /ErOV; J -  6  J  H  7  B W '  O A + Z . L A , ^ 0          E