o ưi[ @sUddlZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZddlZddlZddlmZmZddlmZmZmZddlmZddlZddlmZddlmZmZmZm Z m!Z!m"Z"m#Z#ddl$m%Z%dd l&Terdd lm'Z'dd l(m)Z*dd l+m,Z,neZ'eZ*eZ,zdd l-m.Z.Wn e/ydZ.Ynwde0fddZ1e1Z2ej3dddZ4  d6de ee0e5e6fdede e ee0e feffddZ7iZ8e e e e5e e5e e5fej9fe:d<de e5de e5de e5dej9fddZ; d7de eedeed}|}|durt|ttfr|dur|}||fS|}||fS)a Helper function to route data/content parameters correctly for httpx requests This prevents httpx DeprecationWarnings that cause memory leaks. Background: - httpx shows a DeprecationWarning when you pass bytes/str to `data=` - It wants you to use `content=` instead for bytes/str - The warning itself leaks memory when triggered repeatedly Solution: - Move bytes/str from `data=` to `content=` before calling build_request - Keep dicts in `data=` (that's still the correct parameter for dicts) Args: data: Request data (can be dict, str, or bytes) content: Request content (raw bytes/str) Returns: Tuple of (request_data, request_content) properly routed for httpx N) isinstancebytesstr)r-r. request_datarequest_contentr&r&r'!_prepare_request_data_and_contentJsr4_ssl_context_cachecafilessl_security_levelssl_ecdh_curvec Cstj|d}tjj|_|rt|tr||n|t|rrt|trrz| |t d|W|St ySt dtjddtjd|dY|Styq}zt d|d |d WYd }~|Sd }~ww|S) z Create an SSL context with the given configuration. This is separated from get_ssl_configuration to enable caching. )r6zSSL ECDH curve set to: z is a SSL context use the custom SSL context 2. If ssl_verify is False -> disable SSL verification (ssl=False) 3. If ssl_verify is a string -> use it as a path to CA bundle file 4. If SSL_CERT_FILE environment variable is set and exists -> use it as CA bundle file 5. Else will use default SSL context with certifi CA bundle If ssl_security_level is set, it will apply the security level to the SSL context. SSL contexts are cached to avoid creating duplicate contexts with the same configuration, which reduces memory allocation and improves performance. Args: ssl_verify: SSL verification setting. Can be: - None: Use default from environment/litellm settings - False: Disable SSL verification - True: Enable SSL verification - str: Path to CA bundle file Returns: Union[bool, str, ssl.SSLContext]: Appropriate SSL configuration )rIZSSL_SECURITY_LEVELZSSL_ECDH_CURVENrLF)r6r7r8)r/r9 SSLContextrSr"rNrOr7r8r1rPrQcertifiwherer5rH)rIr7r8r6rR cache_keyr&r&r'get_ssl_configurations.     rX_shared_realtime_ssl_contextcCstdurtatS)z Lazily create the SSL context reused by realtime websocket clients so we avoid import-order cycles during startup while keeping a single shared configuration. N)rYrXr&r&r&r'get_shared_realtime_ssl_contextsrZcCsvt|tr |d}n|S|dkr9|d|}|dkr'|d|dd}|S|d|dd||d}|S|S)Nzkey=&z[REDACTED_API_KEY])r/r1find) error_messageZ key_indexZ next_paramZmasked_messager&r&r'mask_sensitive_info!s     r`cs2eZdZ ddeedeeffdd ZZS)MaskedHTTPStatusErrorNmessagetextcsftt|jj}tj|jtj|jj ||jj |jj dtj |j j|j j |j j dd||_||_dS)N)methodurlheadersr.) status_coder.rf)rbrequestresponse)r`r1rhresuper__init__rbhttpxRequestrdrfr.Responserirgrc)selforiginal_errorrbrcZ masked_url __class__r&r'rk>s" zMaskedHTTPStatusError.__init__NN)__name__ __module__ __qualname__rr1rk __classcell__r&r&rqr'ra=srac@seZdZ      d9deeeejfdeee e e de ffdee dee dedf d d Z  d:deeeejfdeee e e de ffdee dedd ejf d d ZddZddZddZ   d;de deedeedeefddZe         dde d)ejdeeee efdeedeedeeded e fd*d+Zd?d,d-Z e!   d;d.ee"j#deededd eee$e%ffd/d0Z&e!d efd1d2Z'e!  d:deed.ee"j#d e(e e ffd3d4Z)e!   d;deed.ee"j#dedd e$fd5d6Z*e!d ee%fd7d8Z+dS)@AsyncHTTPHandlerNr+ event_hooks. client_aliasrIshared_sessionr cCs*||_||_|j||||d|_||_dS)N)r+ryrIr{)r+ry create_clientclientrz)ror+ryconcurrent_limitrzrIr{r&r&r'rkXs  zAsyncHTTPHandler.__init__r!c Csnt|}tdtj}|durt}tjt|t j r|ndt|t r#|nd|d}t }t j||||||ddS)NSSL_CERTIFICATE ssl_contextrIr{T) transportryr+verifycertrffollow_redirects)rXr"rNrOssl_certificate_DEFAULT_TIMEOUTrx_create_async_transportr/r9rTboolr(rl AsyncClient) ror+ryrIr{ ssl_configrrdefault_headersr&r&r'r|ks&zAsyncHTTPHandler.create_clientc|jIdHdSNr}acloseror&r&r'closezAsyncHTTPHandler.closecs|jSr)r}rr&r&r' __aenter__szAsyncHTTPHandler.__aenter__crrrrr&r&r' __aexit__rzAsyncHTTPHandler.__aexit__reparamsrfrcsH|dur|nt}|p i}|t||jj||||dIdH}|S)N)rrfr)rupdate HTTPHandlerextract_query_paramsr}r$rorerrfrZ_follow_redirectsrir&r&r'r$s  zAsyncHTTPHandler.getFr-jsonstream logging_objfilesr.c  st} z/|dur |j}t|| \} } |jjd|| ||||| | d }|jj||dIdH}||WStjtj fyi|j ||j d}z|j |||||||dIdHW| IdHYS| IdHwtjy}z8t}t|| d}i}t|dd}|dur|jD] \}}||d|<qtjd |d |d d d |dd}~wtjy}z;|durt|d|jIdHt|d|jIdHnt|dt|jjt|dt|jjt|d|jj|d}~wty}z|d}~ww)NPOSTr-rrrfr+rr.rr+ryrer}r-rrrfrriresponse_headers-{}z%Connection timed out. Timeout passed=z , time taken=z secondsdefault-model-namelitellm-httpx-handlerrbmodel llm_providerrfTrbrcrg)timer+r4r} build_requestsendraise_for_statusrlRemoteProtocolError ConnectErrorr|rysingle_connection_post_requestrTimeoutExceptionroundgetattrrfitemsformatrOTimeoutHTTPStatusErrorsetattrriareadr`rcrg Exception)rorer-rrrfr+rrrr. start_timer2r3reqri new_clientrGend_timeZ time_deltaerror_responsekeyvaluer&r&r'posts  $  zAsyncHTTPHandler.postc  z,|dur |j}t||\} } |jjd|| ||||| d} |j| IdH} | | WStjtjfyb|j ||j d} z|j || |||||dIdHW| IdHYS| IdHwtj y}z*i}t|dd}|dur|jD] \}}||d|<q{tjd|dd d |d d}~wtjy}z$t|d |jj|d urt|d|jIdH|t|d|jj|d}~wty}z|d}~ww)NPUTr-rrrfr+r.rrrirConnection timed out after seconds.rrrrgTrbr+r4r}rrrrlrrr|ryrrrrrfrrrOrrrrirgrrcrrorer-rrrfr+rr.r2r3rrirrGrrrr&r&r'putj  $   zAsyncHTTPHandler.putc  r)NPATCHrrrrirrrrrrrgTrbrrr&r&r'patchErzAsyncHTTPHandler.patchc  sDz.|dur |j}t||\} } |jjd|| ||||| d} |jj| |dIdH} | | WStjtjfyd|j ||j d} z|j || |||||dIdHW| IdHYS| IdHwtj y}z$t|d|jj|durt|d|jIdH|t|d|jj|d}~wty}z|d}~ww) NDELETErrrrrgTrb)r+r4r}rrrrlrrr|ryrrrrrirgrrcr)rorer-rrrfr+rr.r2r3rrirrGr&r&r'deletesP  $ zAsyncHTTPHandler.deleter}c sHt||\} } |jd|| |||| d} |j| |dIdH} | | S)zz Making POST request for a single connection client. Used for retrying connection client errors. rr-rrrfr.rN)r4rrr) rorer}r-rrrfrr.r2r3rrir&r&r'rsz/AsyncHTTPHandler.single_connection_post_requestcCs.z t|WdStyYdSwr)asyncioget_running_loop create_taskrrrr&r&r'__del__s  zAsyncHTTPHandler.__del__rcCs tr tj|||dStS)a - Creates a transport for httpx.AsyncClient - if litellm.force_ipv4 is True, it will return AsyncHTTPTransport with local_address="0.0.0.0" - [Default] It will return AiohttpTransport - Users can opt out of using AiohttpTransport by setting litellm.use_aiohttp_transport to False Notes on this handler: - Why AiohttpTransport? - By default, we use AiohttpTransport since it offers much higher throughput and lower latency than httpx. - Why force ipv4? - Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them r)rx_should_use_aiohttp_transport_create_aiohttp_transport_create_httpx_transportrr&r&r'rs z(AsyncHTTPHandler._create_async_transportcCsDddl}ddlm}tjdus||dddurdStddS) z AiohttpTransport is the default transport for litellm. Httpx can be used by the following - litellm.disable_aiohttp_transport = True - os.getenv("DISABLE_AIOHTTP_TRANSPORT") = "True" rNrJTZDISABLE_AIOHTTP_TRANSPORTFalseFzUsing AiohttpTransport...)r"rMrKrOdisable_aiohttp_transportrNrr@)r"rKr&r&r'rs   z.AsyncHTTPHandler._should_use_aiohttp_transportcCs:dtjrdndi}|dur||d<|S|durd|d<|S)aq Helper method to get SSL connector initialization arguments for aiohttp TCPConnector. SSL Configuration Priority: 1. If ssl_context is provided -> use the custom SSL context 2. If ssl_verify is False -> disable SSL verification (ssl=False) Returns: Dict with appropriate SSL configuration for TCPConnector local_addr)0.0.0.0rNr9F)rO force_ipv4)rIrconnector_kwargsr&r&r'_get_ssl_connector_kwargssz*AsyncHTTPHandler._get_ssl_connector_kwargscsddlm}ddlm}tj||d}tj|t dddur"dd}|dur+|}n|d ur1d }t d |durO|j sOt d t |d |||d d St dttd|tradd<tdkritd<tdkrqtd<|fdd|dS)z Creates an AiohttpTransport with RequestNotRead error handling Note: aiohttp TCPConnector ssl parameter accepts: - SSLContext: custom SSL context - False: disable SSL verification rrrJ)rIrZAIOHTTP_TRUST_ENVrTNFzCreating AiohttpTransport...z4SHARED SESSION: Reusing existing ClientSession (ID: ))r}rIZ owns_sessionzDNEW SESSION: Creating new ClientSession (no shared session provided))Zkeepalive_timeoutZ ttl_dns_cacheZenable_cleanup_closedlimitZlimit_per_hostcsttdidS)N)Z connector trust_envr&r r&Ztransport_connector_kwargsrr&r'|s z.)r}rI)+litellm.llms.custom_httpx.aiohttp_transportrrMrKrxrrOZaiohttp_trust_envr"rNrr@closedidrrrrr)rIrr{rrKrZssl_for_transportr&rr'r7sX   z*AsyncHTTPHandler._create_aiohttp_transportcCstjrtddSdS)z Creates an AsyncHTTPTransport - If force_ipv4 is True, it will create an AsyncHTTPTransport with local_address set to "0.0.0.0" - [Default] If force_ipv4 is False, it will return None r local_addressN)rOrrr&r&r&r'rs z(AsyncHTTPHandler._create_httpx_transport)NNNNNNrsNNN) NNNNNFNNNNNNNNFN)NNNNFNr!N),rtrurvrr floatrlrrr1rrr VerifyTypesrkrr|rrrdictrr$rr0LiteLLMLoggingObjectrrrrrrr staticmethodr9rTrrrrrrrrr&r&r&r'rxWs  %    V E E 9  "   Krxc@seZdZ     d&deeeejfdeejdeee e fdee fddZ d d Z   d'd e d ee d ee dee fddZed e dee e ffddZ         d(d e deee e efdeee e efd ee d ee de deeeejfdeee efdedeefddZ       d)d e deee e efdeee e fd ee d ee de deeeejfdefddZ       d)d e deee e efdeee e fd ee d ee de deeeejfdefddZ       d*d e deee e efdee d ee d ee deeeejfde defd d!Zd+d"d#Zdeefd$d%ZdS),rNFr+r}rIdisable_default_headersc Csf|durt}t|}tdtj}|stnd}|dur.|} tj | ||||dd|_ dS||_ dS)NrT)rr+rrrfr) rrXr"rNrOrr(_create_sync_transportrlClientr}) ror+r~r}rIrrrrrr&r&r'rks   zHTTPHandler.__init__cCs|jdSr)r}rrr&r&r'rszHTTPHandler.closererrfrcCs>|dur|nt}|p i}||||jj|||d}|S)N)rrf)rrrr}r$rr&r&r'r$s zHTTPHandler.getr!cCs&ddlm}m}||}t||jS)u Parse a URL’s query-string into a dict. :param url: full URL, e.g. "https://.../path?foo=1&bar=2" :return: {"foo": "1", "bar": "2"} r) parse_qslurlsplit) urllib.parserrrquery)rerrpartsr&r&r'rsz HTTPHandler.extract_query_paramsr-rrrr.rc  Cs:z8t|| \} } |dur|jjd|| |||||| d } n|jjd|| ||||| d} |jj| |d}||WStjyLtjd|dddd tj y}z7|d urot |d t |j t |d t |j nt |j j}t |d |t |d |t |d |j j|d}~wty}z|d}~ww)Nrr)r-rrrfrr.rrrrrrbrrTrbrcrgr4r}rrrrlrrOrrrr`rireadrcrgr)rorer-rrrfrr+rr.rr2r3rrirG error_textr&r&r'rsT      zHTTPHandler.postc  Cs6z6t||\} } |dur|jjd|| ||||| d} n |jjd|| |||| d} |jj| |d} | | WStjyJtjd|dddd tj y} z7|d urmt | d t | j t | d t | j nt | j j}t | d |t | d |t | d | j j| d} ~ wty} z| d} ~ ww)NrrrrrrrrrTrbrcrgr)rorer-rrrfrr+r.r2r3rrirGrr&r&r'rD     zHTTPHandler.patchc  Csz2t||\} } |dur|jjd|| ||||| d} n |jjd|| |||| d} |jj| |d} | WStjyFtjd|dddd tyS} z| d} ~ ww) Nrrrrrrrrr) r4r}rrrlrrOrr)rorer-rrrfrr+r.r2r3rrirGr&r&r'rJs.  zHTTPHandler.putc  Cs6z6t||\} } |dur|jjd|| ||||| d} n |jjd|| |||| d} |jj| |d} | | WStjyJtjd|dddd tj y} z7|d urmt | d t | j t | d t | j nt | j j}t | d |t | d |t | d | j j| d} ~ wty} z| d} ~ ww)NrrrrrrrrrTrbrcrgr)rorer-rrrfr+rr.r2r3rrirGrr&r&r'rnrzHTTPHandler.deletecCs$z|WdStyYdSwr)rrrr&r&r'rs  zHTTPHandler.__del__cCstjrtddSttddS)z Create an HTTP transport with IPv4 only if litellm.force_ipv4 is True. Otherwise, return None. Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them rrZsync_transportN)rOrrrrr&r&r'rs  z"HTTPHandler._create_sync_transport)NNNNFr) NNNNFNNNN)NNNNFNNrr)rtrurvrr rrlrrrr1rkrrr$rrrr0rrrrrrrrrrrr&r&r&r'rs &    = 4 ' 1rrrr{r c Csd}|dur#|D]\}}z ||d|7}Wq ty"Yq wd||}ttdd}|durBddlm}|}ttd|||} | rK| S|durddd |D} || d <tdi| } n tt j d d d |d} |j || t d| S)z Retrieves the async HTTP client from the cache If not present, creates a new client Caches the new client and returns it. N_Zasync_httpx_clientin_memory_llm_clients_cacherLLMClientCachecSi|] \}}|dkr||qSrr&.0kvr&r&r' z*get_async_httpx_client..r{@r)r*)r+r{rrttlr&) rrrrO#litellm.caching.llm_caching_handlerrr get_cacherxrlr set_cacher) rrr{_params_key_namerr_cache_key_namecacher_cached_clienthandler_params _new_clientr&r&r'get_async_httpx_clients>        rc Csd}|dur#|D]\}}z ||d|7}Wq ty"Yq wd|}ttdd}|dur@ddlm}|}ttd|||}|rI|S|dur^dd |D}tdi|} n tt j d d d d } |j || t d| S)z Retrieves the HTTP client from the cache If not present, creates a new client Caches the new client and returns it. rNrZ httpx_clientrrrcSrrr&rr&r&r'r r z%_get_httpx_client..r r)r*)r+r r&) rrrrOrrrrrrlrrr) rrrrrrrrrrr&r&r'_get_httpx_clients6     rrsr)Jrr"r9rCrtypingrrrrrrrr r rUrlZaiohttpr r rrrZ httpx._typesrrOZlitellm._loggingrZlitellm.constantsrrrrrrrZ(litellm.litellm_core_utils.logging_utilsrZlitellm.types.llms.custom_httprZ*litellm.litellm_core_utils.litellm_loggingrrrrZlitellm._versionr rrr(rfrrr1r0r4r5rT__annotations__rHrrSrrXrYrZr`rrarxrZhttpxSpecialProviderrrr&r&r&r's ,   $     - 1  ( B >%  5