o
    ưi)                 U   @   s  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& erwd dl'm(Z( d dl)Z)d dl*Z*d dl+Z+d dl,Z,d d	l-m.Z. d d
l/m0Z0 d dl1Z1d dl1m2Z2 d dl3m4Z4m5Z5m6Z6 erd dl7m8Z8 d dl9m:Z:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZA d dlBmCZCmDZD d dlEmFZF d dlGmHZH d dlImJZJmKZK d dl7m8ZL d dlMmNZNmOZO d dlPmQZQ d dlRmSZSmTZT d dlUmVZV d dlWmXZX d dlYmZZZ d dl[m\Z\m]Z] d dl^m_Z_ d dl`maZambZb d dlcmdZd d d lemfZfmgZg d d!lhmiZi d d"ljmkZkmlZlmmZmmnZn d d#l3moZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d$d%lmZ d$d&lmZmZmZ d$d'lmZ d$d(lmZmZ d$d)lmZmZmZ d$d*lmZmZmZmZmZmZ d$d+lmZ d$d,lmZ d$d-lmZ d$d.lmZmZ d$d/lmZ d$d0lmZ d$d1lmZ d$d2lmZ d$d3lmZmZ d$d4lmZ d$d5lmZ d$d6lmZ d$d7lmZ d$d8lmZ d$d9lmZ d$d:lmZ d$d;lmZ d$d<lmZ d$d=lmZmZ d$d>lmZ d$d?lmZmZ d$d@lmZ d$dAlmZ d$dBlmZ d$dClmZ d$dDlmZ d$dElmZ d$dFlmZ d$d:lmZ d$dGlmZ d$dHlmZ d$dIlmZ d$dJlmZ d$dKlmZ d$dLlmZ d$dMlmZ d$dNlmZ d$d:lmZ d$dOlmZ d$dElmZ  d$dPlmZ d$dQlmZ d$dRlmZ d$dSlmZ d$dTl	m
Z
 d$dUlmZ d$dVlmZ d$dWlmZ d$dXlmZ d$dYlmZ d$dZlmZ d$d[lmZ d$d:lm̐Z d$d\lmZ d$d]lmZ d$d^lm Z  d$d_l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) d$d`l*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 e Z4e Z5e Z6e Z7e݃ Z8e Z9e Z:e Z;e Z<e Z=e Z>e Z?e Z@e ZAe ZBe ZCeʃ ZDe ZEe ZFeă ZGe ZHe
 ZIe ZJe ZKe ZLe ZMe ZNe ZOe ZPe ZQe ZRe ZSe ZTeփ ZUeу ZVeσ ZWe ZXeƃ ZYe߃ ZZe Z[e Z\e Z]e"e^e_e`eself ZaG dadb dbZbG dcdd ddZcG dedf dfZdG dgdh dhZeeFf e2g dddddddddddddddddddddddddddddddddddddddf(die^djedkee dlee^ dmee"egehf  dneeg doeeg dpeeh dqeei dree` dseeh dteeh dueee$  dvee% dwee# dxeeg dyeeg dzee` d{ee^ d|ee"e`e!e. f  d}eeh d~ee dee"e^e`f  deei deei deeh deed  deed  dee^ dee^ dee^ dee^ dee^ deej dee` dee  dee) ded deei de"esepf fPddZkdd Zl	dde"e^e_f die^dee^ fddZmdeei dmee"ege^e*jnf  die^fddZodeei dmee"ege^e*jnf  die^fddZpdme"ege^e*jnf fddZqdme"ege^e*jnf fddZr								ddie^djedqeei dpeeh deea dee deei dmee"ege^e*jnf  fddZs	ddie^de^dee) de e`e^f fddZtdee^ die^deifddZude`de`fddZvd~eee`  deee`  fddZw	dde^de`dee` de`fddZxeFf e2g dddddddddddddddddddddddddddddddddddddddf(die^djedmee"ege^e*jnf  dneeg doeeg dpeeh dqeei dree` dteeh dseeh dueee$  dvee% dwee# dxeeg dyeeg dzee` d{ee^ deed  deed  d|ee"e`e!e. f  d}eeh d~ee dee"e^e`f  deei deeh deei dee) dee` dee^ dee^ dkee dlee^ dee^ dee^ dee^ deej dee  ded deei de"esepf fPddZdd Zydd Zzdd Z{dd Z|e2deqfddZ}e0g ddddddddddddfdeeh dee^ dee^ dee^ dee^ dee^ dȐeid{ee^ ded deeeeqf fdd˄Z~e0g ddddddddddddfdd̜deeh dee^ dee^ dee^ dee^ dee^ dȐeid{ee^ ded deqfdd˄Z~e2g ddddddddddddfdeeh dee^ dee^ dee^ dee^ dee^ dȐeid{ee^ de"eqeeeeqf f fdd˄Z~e2de"evewf fddЄZe2																					dde"e^ee"e^ee"e^eeh f  f  f diee^ deeh deei dyeeg dzeeehehf  deeh dseeh dpeeh dxeeg dee"e^ee^ f  dqeei dree` dee^ dneeg doeeg d{ee^ dee^ dee^ dee^ deej dee^ f,ddׄZdؐe^dee"e.e+f  fddڄZde"ee^ef ee f fdd܄Zdؐe^dee"e.e+f  fddބZ	ddߐe^diee^ dee^ de(fddZe2			ddߐe^diee^ dee^ dee^ de(f
ddZe2dexfddZe2												ddie^de.dee^ dee^ d|eed  deeed   dneeh d{ee^ dee^ dee^ dee^ deeh de"exeeeexf f fddZe2de'fddZe2																ddie^dߐe^dee"e^e`f  dee^ dee^ dee^ dee^ dee^ deeh dee` dmee"ege*jnf  d|ee^ deeh dee^ dee` dee^ deei de"e'eeee'f f f$ddZ			dde`deed  dee^ dee fddZdd  Zdd Z	ddejdjee devfddZ				ddejdjeej ded dee"esevf  fdd	Zdaee ed
< dd Zde^defddZdS (      N)futures)FIRST_COMPLETEDThreadPoolExecutorwait)deepcopy)partial)TYPE_CHECKINGAnyAsyncIteratorCallable	CoroutineDictIterableListLiteralMappingOptionalTupleTypeUnioncastget_args)uuid)ClientSession)	BaseModel)overload)client)exception_typeget_litellm_paramsget_optional_params)Logging),DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT)LiteLLMUnknownProvider)CustomLogger)run_async_function)calculate_request_durationget_audio_file_for_health_check)tracer)ProviderSpecificHeaderUtils)_create_health_check_response_filter_model_params)mock_embeddingmock_image_generation)get_content_from_model_response)
BaseConfigBaseImageGenerationConfig)#convert_model_response_to_streaming)BedrockModelInfo)CohereModelInfo)AsyncHTTPHandlerHTTPHandler)JSONProviderRegistry)VertexAIModelRouteget_vertex_ai_model_route)_realtime_health_check)get_secret_boolget_secret_str)GenericLiteLLMParams)CustomPricingLiteLLMParamsModelResponseStreamRawRequestTypedDictStreamingChoices)&ChoicesCustomStreamWrapperEmbeddingResponseMessageModelResponseProviderConfigManagerTextChoicesTextCompletionResponseTextCompletionStreamWrapperTranscriptionResponseUsage_get_model_info_helper/add_provider_specific_params_to_optional_params#async_mock_completion_streaming_obj convert_to_model_response_objectcreate_pretrained_tokenizercreate_tokenizerget_api_keyget_llm_provider!get_non_default_completion_params$get_non_default_transcription_paramsget_optional_params_embeddingsget_optional_params_image_gen!get_optional_params_transcriptionget_requester_metadata
get_secretget_standard_openai_paramsmock_completion_streaming_objpre_process_non_default_paramsread_config_argsshould_run_mock_completionsupports_httpx_timeouttoken_counter validate_and_fix_openai_messagesvalidate_and_fix_openai_toolsvalidate_and_fix_thinking_param$validate_chat_completion_tool_choicevalidate_openai_optional_params   )verbose_logger)disable_cacheenable_cacheupdate_cache)safe_deep_copy)async_completion_with_fallbackscompletion_with_fallbacks)add_system_prompt_to_messagesget_completion_messages#update_messages_with_model_file_ids)custom_promptfunction_call_promptmap_system_message_pt	ollama_ptprompt_factory stringify_json_tool_call_content)ChunkProcessor)AnthropicChatCompletion)AzureAudioTranscription)AzureChatCompletion_check_dynamic_azure_params)AzureOpenAIO1ChatCompletion)AzureTextCompletion)AzureAnthropicChatCompletion)AzureAIEmbedding)BedrockConverseLLM
BedrockLLM)BedrockEmbedding)BedrockImageEdit)BedrockImageGeneration)BytezChatConfig)ClarifaiConfig)CodestralTextCompletion)handler)BaseLLMAIOHTTPHandler)BaseLLMHTTPHandler)	CustomLLMcustom_chat_llm_router)DatabricksEmbeddingHandler)aleph_alphapalm)get_api_key_from_env)GroqChatCompletion)HerokuChatConfig)HuggingFaceEmbedding)LemonadeChatConfig)
completion)OCIChatConfig)	oobabooga)OpenAITextCompletion)OpenAIImageVariationsHandler)OpenAIChatCompletion)OpenAIAudioTranscription)OpenAILikeChatHandler)OpenAILikeEmbeddingHandler)OVHCloudChatConfig)PredibaseChatCompletion)SagemakerChatHandler)SagemakerLLM)GenAIHubOrchestration)vertex_ai_non_gemini)	VertexLLM)GoogleBatchEmbeddings)VertexImageGeneration)VertexMultimodalEmbedding)VertexAIPartnerModels)VertexEmbedding)VertexAIGemmaModels)VertexAIModelGardenModels)WatsonXChatHandler)IBMWatsonXMixin)AnthropicThinkingParam)ChatCompletionAssistantMessageChatCompletionAudioParamChatCompletionModality$ChatCompletionPredictionContentParamChatCompletionUserMessageHttpxBinaryResponseContentOpenAIModerationResponseOpenAIWebSearchOptions)	AdapterCompletionStreamWrapperChatCompletionMessageToolCallCompletionTokensDetails	FileTypesHiddenParamsLlmProvidersPromptTokensDetailsProviderSpecificHeaderall_litellm_paramsc                   @   sX   e Zd Zddddejdddee dee dee dee dee	eef  f
d	d
Z
dS )LiteLLMNX  )api_keyorganizationbase_urltimeoutmax_retriesdefault_headersr   r   r   r   r   c                C   s   t  | _t| jd d| _d S )N
router_obj)localsparamsChatchat)selfr   r   r   r   r   r    r   C/home/app/Keep/.python/lib/python3.10/site-packages/litellm/main.py__init__6  s   
zLiteLLM.__init__)__name__
__module____qualname__litellmnum_retriesr   strfloatintr   r   r   r   r   r   r   5  s$    r   c                   @   s   e Zd Zdee fddZdS )r   r   c                 C   sL   || _ | j dddu r| j d t| j |d| _d S t| j |d| _d S )NacompletionFTr   )r   getpopAsyncCompletionsZcompletionsCompletionsr   r   r   r   r   r   r   E  s   zChat.__init__N)r   r   r   r   r	   r   r   r   r   r   r   D  s    r   c                   @   (   e Zd Zdee fddZdddZdS )r   r   c                 C      || _ || _d S Nr   r   r   r   r   r   r   Q     
zCompletions.__init__Nc                 K   sn   |  D ]	\}}|| j|< q|p| jd}| jd ur*| jjd||d| j}|S td||d| j}|S Nmodelr   messagesr   )itemsr   r   r   r   r   r   r   kwargskvresponser   r   r   createU  s   
zCompletions.creater   r   r   r   r   r	   r   r   r   r   r   r   r   P      r   c                   @   r   )r   r   c                 C   r   r   r   r   r   r   r   r   c  r   zAsyncCompletions.__init__Nc                    s|   |  D ]	\}}|| j|< q|p| jd}| jd ur.| jjd||d| jI d H }|S td||d| jI d H }|S r   )r   r   r   r   r   r   r   r   r   r   g  s   
zAsyncCompletions.creater   r   r   r   r   r   r   b  r   r   r   r   	functionsfunction_callr   temperaturetop_pnstreamstream_options
max_tokensmax_completion_tokens
modalities
predictionaudiopresence_penaltyfrequency_penalty
logit_biasuserresponse_formatseedtoolstool_choiceparallel_tool_callslogprobstop_logprobsreasoning_effort)noneZminimallowmediumhighZxhighdefault	verbosity)r  r  r  safety_identifierservice_tierr   api_versionr   
model_listextra_headersthinkingweb_search_optionsshared_sessionr   enable_json_schema_validationreturnc)           :         s  |) dd}*|) dd}+|+du rt|+|| I dH  t },|) dd}-|) dd}.t|.trj|.j|) dd|)|drj|.j| ||)|) dd|) d	d||) d
d|) dddI dH \} }}/|durjt|dkrjd}|'durzt	
dt|' d nt	
d i d| d|d|d|d|d|d|d|d|d|	d|
d|d|d|d|d |d!|i d"|d#|d$|d%|d&|d'|d(|d)|d*|d+|d,|d-| d.|!d/|"d0|#d1|d2|||$d|%|&|'|(d3}0|-du rt| |-|0 d-dd4\}/}-}/}/|*ptj}*|*dur/tdAi |0d5d|*i|)iI dH }1|1du r-td6|1S |) d7}2|) d8}3|) d9}4|) d}+|2rVt|3|4|+d:rVt|2I dH  zqttfi |0|)}5t }6t|6j|5}7|,d|7I dH }8t|8tst|8trt|8trtdAi |8}1|8}1nt|8r|8I dH }1n|8}1|-d;ks|-d<krt|1trt j|1t d=}1t|1tr|1j |,d> |1W S  ty }9 z|-pd?}-t!| |-|9|0|)d@d}9~9ww )Ba  
    Asynchronously executes a litellm.completion() call for any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly)

    Parameters:
        model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/
        messages (List): A list of message objects representing the conversation context (default is an empty list).

        OPTIONAL PARAMS
        functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list).
        function_call (str, optional): The name of the function to call within the conversation (default is an empty string).
        temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0).
        top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0).
        n (int, optional): The number of completions to generate (default is 1).
        stream (bool, optional): If True, return a streaming response (default is False).
        stream_options (dict, optional): A dictionary containing options for the streaming response. Only use this if stream is True.
        stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens.
        max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity).
        max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
        modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request. You can use `["text", "audio"]`
        prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content.
        audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"]
        presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far.
        frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
        logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
        user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
        metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc.
        api_base (str, optional): Base URL for the API (default is None).
        api_version (str, optional): API version (default is None).
        api_key (str, optional): API key (default is None).
        model_list (list, optional): List of api base, version, keys
        timeout (float, optional): The maximum execution time in seconds for the completion request.

        LITELLM Specific Params
        mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
        custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock"
    Returns:
        ModelResponse: A response object containing the generated completion and associated metadata.

    Notes:
        - This function is an asynchronous version of the `completion` function.
        - The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
        - If `stream` is True, the function returns an async generator that yields completion lines.
    	fallbacksNmock_timeoutTcustom_llm_providerlitellm_logging_obj	prompt_id)r  non_default_paramsr  prompt_variablesprompt_labelprompt_version)r   r   r   r  r!  r  r"  r#  r   uA   🔄 SHARED SESSION: acompletion called with shared_session (ID: )uA   🔄 NO SHARED SESSION: acompletion called without shared_sessionr   r   r   r   r   r   r   r   r   r   stopr   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r	  deployment_idr   r  r   r  r
  r  )r  r  r   r  r  r  r  r   r  api_baser   z]No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details.
mock_delaymock_responsemock_tool_calls)r*  r+  r  text-completion-openaitext-completion-codestralresponse_objectZmodel_response_object)loopopenair   r  original_exceptioncompletion_kwargsextra_kwargsr   )"r   _handle_mock_timeout_asyncasyncioget_event_loop
isinstanceLiteLLMLoggingObj"should_run_prompt_management_hooksZ async_get_chat_completion_promptlenrh   debugidrS   r   model_fallbacksrm   	Exceptionr_   sleepr   r   contextvarscopy_contextrunrun_in_executordictrE   iscoroutinerH   OpenAITextCompletionConfig%convert_to_chat_model_response_objectrB   Zset_logging_event_loopr   ):r   r   r   r   r   r   r   r   r   r   r%  r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r	  r&  r
  r  r  r  r   r  r   r  r  r  r  r  r  r   r  r  r0  r  r  _r4  r   r)  r*  r+  funcctxfunc_with_contextinit_responseer   r   r   r   t  sj  a






	
 !"#
+










r   c              
   C  s   z(t d|   t| r| I d H } | 2 z3 d H W }t d|  |V  q6 W d S  ty@ } z|p4d}t|||dd }~ww )Nz'received response in _async_streaming: zline in async streaming: r1  )r   r  r3  )print_verboser7  rG  r@  r   )r   r   r  argslinerO  r   r   r   _async_streaming  s$   

rS  r*  r  c              	   C   s  t | tr,t | tjr| tjt| ddt| dt| t| d|p!d|tj	dddd	t | trC| d
krCtj
dt| d|p>d|dt | trZ| dkrZtjdt| d|pUd|dt | trq| dkrqtjdt| d|pld|dt | tr| drtjd| d|tj	dddd	d S d S )Nstatus_code  textllm_providerr1  POSTzhttps://api.openai.com/v1/)methodurl)rT  messagerW  r   requestzlitellm.RateLimitErrorzthis is a mock rate limit errorr[  rW  r   z"litellm.ContextWindowExceededErrorz,this is a mock context window exceeded errorzlitellm.InternalServerErrorz$this is a mock internal server errorz Exception: content_filter_policyi  azure)r9  r@  r1  APIErrorr   MockExceptiongetattrr   httpxRequestZRateLimitErrorZContextWindowExceededErrorZInternalServerError
startswithr*  r   r  r   r   r   !_handle_mock_potential_exceptions  sf   


	


rf  r  c                 C   s0   | du r|d urt | tjdd|dd S d S NTzThis is a mock timeout errorr1  r]  )_sleep_for_timeoutr   Timeoutr  r   r   r   r   r   _handle_mock_timeout  s   rk  c                    s8   | du r|d urt |I d H  tjdd|dd S d S rg  )_sleep_for_timeout_asyncr   ri  rj  r   r   r   r6    s   r6  c                 C   sb   t | trt|  d S t | trtt|  d S t | tjr-| jd ur/t| j d S d S d S r   )r9  r   timerA  r   rb  ri  connectr   r   r   r   rh    s   

rh  c                    sv   t | trt| I d H  d S t | tr!tt| I d H  d S t | tjr7| jd ur9t| jI d H  d S d S d S r   )r9  r   r7  rA  r   rb  ri  rn  ro  r   r   r   rl    s   

rl  FThis is a mock requestr+  c
              
   K   s  z4|
 dpd}|du rd}t||	| d |dur!|j|dd t|ts+t|tr2t|| |d tttt	t
tf |}t|trP|d	rPtjd
d| dd}|
 dd}|dura|sat| t|t	rnt
d!i |W S t|t
r||sx|W S t|}t
 }|du rt }|
 dddu rtt||| |d| d|dW S tt||| |d| d|dW S t|tjr|tt|}|du r||jd j_ng }t|D ]}tjj|tjj|ddd}|| q||_tt |_| |_|rdd |D |jd j_ t!|dt"t#t$t#t$ d ztjj%| d\}}}}||j&d< W n
 ty%   Y nw |dur3|j'|ddd |W S  tyR } zt|t(j)rG|td *|d}~ww )"aS  
    Generate a mock completion response for testing or debugging purposes.

    This is a helper function that simulates the response structure of the OpenAI completion API.

    Parameters:
        model (str): The name of the language model for which the mock response is generated.
        messages (List): A list of message objects representing the conversation context.
        stream (bool, optional): If True, returns a mock streaming response (default is False).
        mock_response (str, optional): The content of the mock response (default is "This is a mock request").
        mock_timeout (bool, optional): If True, the mock response will be a timeout error (default is False).
        timeout (float, optional): The timeout value to use for the mock response (default is None).
        **kwargs: Additional keyword arguments that can be used but are not required.

    Returns:
        litellm.ModelResponse: A ModelResponse simulating a completion response with the specified model, messages, and mock response.

    Raises:
        Exception: If an error occurs during the generation of the mock completion response.
    Note:
        - This function is intended for testing or debugging purposes to generate mock completion responses.
        - If 'stream' is True, it returns a response that mimics the behavior of a streaming completion.
    r   FNrp  rj  zmock-key)inputr   re  zException: mock_streaming_errorz&This is a mock error raised mid-stream	anthropici  )r[  rW  r   rT  r)  T)r*  r   r   r1  completion_streamr   r  logging_objr   Z	assistant)contentrole)indexr[  c                 S   s   g | ]	}t d i |qS )r   )r   ).0Z	tool_callr   r   r   
<listcomp>}  s    z#mock_completion.<locals>.<listcomp>usageprompt_tokenscompletion_tokenstotal_tokensr   r  zmy-secret-keyzmy-original-responserq  r   original_responsez$Mock completion response failed - {}r   )+r   rk  Zpre_callr9  r   r@  rf  r   r   rF  rE   r>   rd  r   r`  rm  rA  r1   rB   rN   r\   choicesr[  rv  rangeutilsrA   rD   appendr   createdr   
tool_callssetattrrK   r"   r!   rS   _hidden_params	post_callr1  r_  format)r   r   r   r   r*  r+  r  loggingr  r   r   Zis_acompletionZ
time_delaymodel_responseZ_all_choicesi_choicerJ  rO  r   r   r   mock_completion  s   $




r  c              
   C   s   i }zBt tt| |d}|dd u r$| dr$| dd} d}||d< |d ur;|dkr@d|d< | dd} W || fS W || fS W || fS  ty{ } z+td	| | drn| dd} d}||d< W Y d }~|| fS W Y d }~|| fS d }~ww )Nr   r  modez
responses/ 	responsesxaizError getting model info: {})
r   rF  rL   r   rd  replacer@  rh   r=  r  )r   r  r  
model_infor  rO  r   r   r   responses_api_bridge_check  sB   

r  c                 C   s4   | dkrdS | dks| dks| dkrd|  v S dS )Nrr  Tazure_aibedrock	vertex_aiclaudeF)lowerr  r   r   r   r   _should_allow_input_examples  s   r  toolc                 C   sH   |   }|dd  |d}t|tr"|  }|dd  ||d< |S )NZinput_examplesfunction)copyr   r   r9  rF  )r  Z	tool_copyr  r   r   r   _drop_input_examples_from_tool  s   

r  c                 C   sB   | d u rd S g }| D ]}t |tr|t| q
|| q
|S r   )r9  rF  r  r  )r  Zcleaned_toolsr  r   r   r   _drop_input_examples_from_tools  s   
r  r   r  c                 C   sn   d| i}t jD ]}||}|dur|||< q|r5t|tr5dD ]}||v r4|| dur4||||  q |S )a"  Build a complete model cost entry from kwargs and model_info.

    Collects all CustomPricingLiteLLMParams fields present in kwargs and
    merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
    so that register_model() receives the full pricing configuration.
    Zlitellm_providerN)r  Zsupports_prompt_cachingr   )r=   Zmodel_fieldsr   r9  rF  
setdefault)r  r   r  entry
field_namevaluekeyr   r   r   _build_custom_pricing_entry  s   

r  c)                    sG   du rt dt|d}t|d}t|d}t|d}t|&d}&t }*|)dd	}+|+s|rd
dlm	}, d
dl
m}- d
dlm}. ttt|.  |}/|-j|/dr|,di d d|d| d|!d|d|d|d|d|d|d|d|
d|	d|d|d|d|d|d |d!|d"|d#|d$|d%|d&|d'|d(|d)|d*|d+|d,|d-|d.|"d/|#d0|$d1|%d2|d3|&d4|d5|'d6|(|)S |)d7d}0|)d8d}1|)d9d}2ttt |)d:d}3|)d;d<}4|)d=d}5|)d>d	}6|)d?d}7|)d@d}8|)dAd}9|)dBd}:|)dCd};|)dDd}<|)dEd}=ttt |)dFd}>|)dGdp>|}?|)dHd}@|)dId}A|)dJd}B|?du rXi }?|durb|?| tjdurztj }C|?|C W n ty }D ztdK|D  W Y d}D~Dnd}D~Dww |)dLd}E|)dMd}F|)dNd}G|)dOd}H|)dPd}I|)dQd}J|)dRd}K|)dSd}L|)dTd}M|)dUd}N|)dVd}O|)dWd}P|)dXd}Q|)dYd}R|)dZd}S|)d[d}T|)d\d}U|)d]d}V|)d^d}W|)d_d}X|)d`d	}Y|)dad	}Z|)dbd	}[|)dcd}\|)ddd	}]ttt |)ded}^ttt |)dfd}_|)dgd}`t||@pHd	|A|Bdh}t|)di}ai }bt |8t!ry|8j"|^|adjry|8j# ||a|^|_|)dkd|)dlddm\ }}c|`rt$||`dndo}!z8|"dur|"}0|Edur|E}Ftt!|8}d|=ptj%}=|=durt&di |*W S |%durƇ fdpdq|%D }etj'ddr|ei|*W S tj(rՈ tj(v rtj(   t) }ft*|fdst+  |)dtd	dnu rdt}7|dur| dt}7t, |7|0|$du\ }7}g}0t-|7 dvst.|d}|>dur|?t/j0|>|7dw |fdur4t1|fdxr4|7|fj2d?< |)dyd|fj2dz< |p?|)d{d<p?d<}t |t3j4rSt5|7sS|j6pQd<}nt |t3j4s^t7|}|Kdurh|Ldusm|Mdur~t8|7 d|  t9|7|)|;d}i i }h|Os|Ps|Qs|Rs|Sr i i}h|Or|O|h  dV< |Pr|P|h  dW< |Qr|Q|h  dX< |Rr|R|h  dY< |Sr|S|h  dZ< |)d~rt:||)dCi dAdtt;tt;ttf f |)d~d}d}i|7dur|7ddq t<D v rt=j> t<|7d}i|idur|ij?|d}|Vdurt |Vtr|Vd	u rt@|d}|gdur$|g}$i d| d|!d|d|d|d|d|d|d|
d|	d|d|d|d|d|d |d!|i d d?|7d"|d#|d$|d%|dM|Fd'|d(|d/|#d&|d|d*|d3|&d4|d,|d-|d|)di}jtAdi |j|a}ctB |j|a|7|)ddndn|id}ktjCr|cddr|cd}ltD||ld}tEdi db|[d0|$d;|4d=|5d>|6d?|7d7|0d|)dddtj(d|9dB|:dC|;dD|<d[|Td|]dT|MdR|KdU|NdS|LdN|Gd`|)d`d|)ddI|)dId^|Wd|)dd|)dd\|Ud|hd|)dd_|Xd|)dde|^df|_dQ|Jd|)ddd|)dd	d/|#d|)dd|)dd|)dd|)dd|)dd|)dd|)ddM|Fd|d|)dd	d|)dd|)d}btt!|djF ||k|b|7d |1s|2s|3r|)d:d tG ||||1|2|d|[|)dd|7|3|dW S tH |7|d\}; |;ddkrd
dlImJ}m |mjK ||?|f|$|0|[|d|c|b||\|7tL |dW S |7dtkrNd	}n|\durt |\tMjNst |\tMjOrtPd/|#i|\d}ntQdp$dt}o|0p/tjRp/tQd}0|#p>tjSp>tTdp>tjU}#|$pRtjVpRtjWpRtTdpRtTd}$|cdi ddpatTd}p|bdd}q|?pmtjX}?|durw||cd2< |Fdur|F|cdM< tY jZ drtjY[ }r|r\ D ]\}s}t|s|cvr|t|c|s< qt]jKdi d d|dG|?d0|$d7|0d/|#d|nd|pd|fdt^d|cd|bd=|5d|ddb|[d|dc|\d?|7}unXtj_[ }r|r\ D ]\}s}t|s|cvr|t|c|s< qt`jKdi d d|dG|?d0|$d7|0d/|#d|od|nd|pd|qd|fdt^d|cd|bd=|5d|ddb|[d|dc|\}u|cdd	rK|dja||$|u|?|#|0dd |uW S |7dkr%tTdpYdt}o|0pdtjRpdtTd}0|0du rnt d|#pxtjSpxtTd}#|$ptjVptjWptTdptTd}$|cdi ddptTd}p|bdd}q|?ptjX}?|dur||cd2< tj_[ }r|r\ D ]\}s}t|s|cvr|t|c|s< qtbjKdi d d|dG|?d0|$d7|0d/tt|#d|od|pd|qd|fdt^d|cd|bd=|5d|ddb|[d|dc|\}u|cdd	s|[dnu r"|dja||$|u|?|#|0dd |uW S |7dkrz?tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|d|i}uW |uW S  ty }D z|dja||$t|DdG|?id |Dd}D~Dww |7dk	rd
dldme}v |vf }w|wdkrd
dlgmh}x |vi|0}0|0du rt d|vj|$}$|xjK ||0|$|f|d|c|b||[||?ptjXd}u|uW S d k v 	rc|vi|0}0|0du rt dÃ|vj|$}$|0	r|0ld|}0|0mdġ	sd|0v 	r|0nddơ}y|yd
 d }0n|0d }0|0d }0tojKdi d d|d7|0db|[dtjpd|fdt^d|cd|bd=|5dtL d0|$d|ddG|?d|dc|\d?|7}u|cdd		sV|[dnu 	r^|dja||$|udǍ |u}u|uW S |vi|0}0|vj|$}$|?	prtjX}?|du	r|||cd2< dȈ v 	rtq|d}z:tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|}uW n t	y }D z|dja||$t|DdG|?id |Dd}D~Dww |cdd		r|dja||$|udG|?id |uW S |7dk
sdʈ v 
sdˈ v 
s|7tjrv 
r|)d`dnu 
rdtM_s|0
p$tjR
p$tQd̓
p$tQd΃
p$d}0dtM_S|$
p6tjV
p6tjt
p6tQdЃ}$|?
p<tjX}?|du
rF||cd2< tju[ }r|r\ D ]\}s}t|s|cv
r\|t|c|s< 
qOtjv
rftjvtM_vtw|d
k
rd|d
 v 
rt |d
 d tx
r|d
 d }zn
dҠyddq |D }ztzjK ||ft^|$|7|0|[|\|d|c|b|5|dԍ}{|cdd	d	u 
r|[d	u 
r|Yd	u 
rtu j{|{|fdՍ}{|cdd	
s|[dnu 
r|dja||$|{dG|?id |{}u|uW S |7dkr;z?tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|d|i}uW |uW S  ty: }D z|dja||$t|DdG|?id |Dd}D~Dww |7dkrz?tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|d|i}uW |uW S  ty }D z|dja||$t|DdG|?id |Dd}D~Dww |7dkrz?tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|d|i}uW |uW S  ty }D z|dja||$t|DdG|?id |Dd}D~Dww |7dkr^z?tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|d|i}uW |uW S  ty] }D z|dja||$t|DdG|?id |Dd}D~Dww |7dkr|0pptjRpptQdۃppd}0|$ptjVptj|ptQd݃}$|?ptjX}?tj}[ }r|r\ D ]\}s}t|s|cvr|t|c|s< qtcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?|7d|dG|?dtL d0|$d|ddc|\}u|uW S |7dkrM|0ptjRptQd߃}0|$ptjVptQd}$|?ptjX}?tj~[ }r|r\ D ]\}s}t|s|cvr|t|c|s< qtcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?|7d|dG|?dtL d0|$d|ddc|\}u|uW S |7dkrtjj |0|$|?|cd\}0}$}?|0pjtjRpjtTd}0|0du rttd|?pytjX}?tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?|7d|dG|?dtL d0|$d|ddc|\d|i}u|uW S |7dkr6|$ptjVptjptQdptQd}$|?ptjXpi }?z?tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|d|i}uW |uW S  ty5 }D z|dja||$t|DdG|?id |Dd}D~Dww |7dkr|?p@tjX}?tj[ }r|r\ D ]\}s}t|s|cvrW|t|c|s< qJtjKdi d d|dG|?d|fdb|[d|dd|cd|bd|d5|'dc|\d?|7dtL d0|$d7|0d|}u|uW S |7dkr|0ptjRptQd̓ptQd΃pd}0|$ptjVptjtptQdЃ}$|?ptjX}?|dur||cd2< tjK ||?|f|$|0|[|d|c|b||\|7tL |d}u|uW S |7dkrJ|$ptjptTdptjV}$|0ptjRptTdpd}0tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|d|i}u|dja||$|udǍ |uW S |7dkr|$pYtTdpYtjV}$|0pgtjRpgtTdpgd}0tcjKdi d d|d7|0d?|7d|fdtL d|dd|cd|d|bd5|'db|[d|d0|$dG|?dc|\d|i}u|dja||$|udǍ |uW S |7dkr|0ptjRptTd}0tcjKdi d d|d7|0d?|7d|fdtL d|dd|cd|d|bd5|'db|[d|d0|$dG|?dc|\d|i}u|dja||$|udǍ |uW S  tjv sc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7dksc|7d ksc|7tjv sct|7scd v r|0putjRputQd̓putQd΃pud}0|IptjvptQdpd}I|ItM_v|$ptjVptjtptQdЃ}$|?ptjX}?|7dkrd
dlm}| d
dlm}} || }~|~j }|}|}|r|| |}|dur||cd2< tjr|:durt|:}|dur||cdB< tj[ }r|r\ D ]\}s}t|s|cvr|t|c|s< qtd}z~|rBtcjKdi d d|d7|0d?|7d|fdtL d|dd|cd|d|bd5|'db|[d|d0|$dG|?dc|\d|i}un=tjKdi d d|dG|?d|fdt^d0|$d7|0db|[d|dd|cd|bd=|5d|d|hdc|\dP|Id?|7d5|'}uW n ty }D z|dja||$t|DdG|?id |Dd}D~Dww |cdd	r|dja||$|udG|?id |uW S |7dkr|$ptjVptQd}$|0ptjRptQd	pӐd
}0tcjKdi d d|d7|0d?|7d|fdtL d|dd|cd|d|bd5|'db|[d|d0|$dG|?dc|\d|i}u|uW S d v s$|7dks$ tjv rz|$p9tjp9tjVp9tQdp9tQd}|0pItjRpItQdpId}0|hpOtjp}ht ||0|ft^|c|b|5tL ||d|h|[|?d}f|cdd	dnu ru|dja|||fdǍ |f}u|uW S d  v s|7d ks tjv r	 |uW S |7dkr|$ptjptjVptjd}$|hptjp}h|0ptjRptQdptQdpÐd}0td}|0dur|s|0mds|0d7 }0n	|rtd tcjK |||[|0|f|c|b|'d||?tL |$|dd}u|uW S |7dkr|$ptjptjVptjd}$|hptjp}h|0p4tjRp4tQdp4tQdp4d}0td}|0durM|sM|0mdġsM|0d7 }0n	|rVtd tjKdi d d|d7|0db|[dtjpd|fdt^d|cd|bd=|5dtL d0|$d|ddG|?d|dc|\d?|7}u|cdd	s|[dnu r|dja||$|udǍ |u}u|uW S |7dkr|$ptjptQdptjV}|0ptjRptQdpАd }0t ||0|ft^|c|b|5tL ||dd!}ud|cv r|cd dnu rt|u d|dd"}u|cdd	s|[dnu r|dja||$|udǍ |u}u|uW S |7d#kro|$p,tjp,tQd$p,tQd%p,tjV}|0p<tjRp<tQd&p<d'}0tjK ||0|ft^|c|b|5tL tj||dd(}fd|cv rj|cd dnu rjt|f d#|dd"}u|uW S |f}u|uW S |7d)ks{|7d*kr@|$ptjptTd+ptTd,ptjV}t }td-|  |d.kr|0ptjRptTd/pd0}0d1 v rÈ d1d2 n|0ptjRptTd/pӐd3}0|?ptjXpi }?|?du ri }?|dur|?| td4  d5|0  td6|i  tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?d)d|dG|?dtL d0|d|id|d}u|uW S |7d7kr|$pUtjpUtQd8pUtjV}|0petjRpetQd9ped:}0tjK ||0|ft^|c|b|5tL ||dd7|hd;}f|f}u|uW S |7d<kr|$ptjptTd=ptjV}$|0ptjRptTd>pd?}0tjK ||0|ft^|c|b|5tL |$|d||7|hd@}u|uW S |7dAkr|$ptjptjdBptjdCptjV}|?ptjX}tcjK |||f||0|[|d|c|b||\|7tL |d}u|uW S |7dDkrtcjK ||?|f|$|0|[|d|c|b||\|7tL |d}u|uW S |7dEkrm|$p+tTdFp+tjV}$|0p1dG}0tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd|dc|\d?|7dtL d|d|i}u|uW S |7dHkrdH}7tjK ||f|0t^|c|bd|5tL |ddI}fd|cv r|cd dnu rt|f dH|dd"}u|uW S |f}u|uW S |7dJkr|0ptjRptdK}0|$ptjVptjptQdL}$|?ptjX}?ztcjK |||[|0|f|c|bdJ||?tL |$|d|\dM}uW n ty }D z|dja||$t|DdG|?id |Dd}D~Dww |cdd	r|dja||$|udG|?id |uW S |7dNkr[tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd|dc|\d?|7dtL d|d|i}u|uW S |7dOkr-|0pptjRpptTdPppdQ}0|$ptjVptjptTdRptTdS}$tQdTpdU}tQdVpdW}||dX}|?ptjX}|r|| |}?tj[ }r|r\ D ]'\}s}t|sdkrd|cv r|c|s |t q|t|c|s< q|s|cvr|t|c|s< q |dY|c}tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?dOd|dG|?dtL d0|$d|ddc|\}u|dja|tMjV|udǍ |uW S |7dZkr|0pBtjRpBtTd[pBd\}0|$pNtjVpNtQd]}$tQd^pWdU}tQd_p`dW}||d`}|?pltjX}|ru|| |}?tj[ }r|r\ D ]'\}s}t|sdkrd|cv r|c|s |t q|t|c|s< q|s|cvr|t|c|s< q |dY|c}tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?dZd|dG|?dtL d0|$d|ddc|\}u|dja|tMjV|udǍ |uW S |7dksda v s tjv r	 	 |uW S |7dbkrt dc|7ddks"|7dekr|cdfdp:|cdgdp:tjp:tQdh}|cdidpS|cdjdpStjpStQdk}|cdldph|cdmdphtQdn}|$pxt pxtQdopxtjV}|0ptjRptQdp}0t|cpi }tjKdi d d|d|fdt^d|d|bd=|5dtL di|df|dl|dq|d|ddb|[d|d?|7dc|\d7|0d2|?}u|uW S |7drkr|cdfdp|cdgdptjptQdh}|cdidp|cdjdptjptQdk}|cdldp |cdmdp tQdn}|0p,tjRp,tQds}0t|cp2i }t |bdt}|tjkrtjKdi d d|d|fdt^d|d|bd=|5dtL d7|0di|df|dl|d|ddb|[dG|?d|hd|dc|\}fnk|tjkrtjKdi d d|d|fdt^d|d|bd=|5dtL di|df|dl|dqdd|ddb|[d|d?|7dc|\d7|0d2|?}fn|tjkrtjKdi d d|d|fdt^d|d|bd=|5dtL d7|0di|df|dl|d|ddb|[dG|?d|hd|dc|\}fn|tjkratjKdi d d|d|fdt^d|d|bd=|5dtL d7|0di|df|dl|d|ddb|[dG|?d|hd|dc|\}fn|tjkrd
dulm} | }||bdf< ||bdi< ||bdl< tcjKdi d d|d|d|fd|d|bdtL d0dd7|0d|ddb|[d|dc|\d?drd|dG|?pi }fn2tjK ||ft^||b|5tL ||||d|[dv}fd|cv r|cd dnu r|[d	u rt|f dr|dd"}u|uW S |f}u|uW S |7dwkru|cddp|cdxdptjÐptQdy}|du rt dz|0p5|cd7dp5|cd.dp5tjRp5tQd{}0|$pEtjVpEtjĐpEtQd|}$tjK ||ft^|c|b|5tL |d|[|0|h|$||d}}d|cv rp|cd dnu rp|[d	u rp|W S |}u|uW S |7d~kr|0p|cd7dp|cd.dptjRpd}0|$ptjVptQd}$tj|d}tjK ||t^|c|b|5tL |d|[|0|h|$|d}d|cv r|cd dnu r|[d	u r|W S |}u|uW S |7dkrtcjK |||[|0|f|c|bd||?tL |$|d|\dM}f|f}u|uW S |7dkrtjK ||ft^|c|b|h|U|5tL |d|[d}f|f}u|uW S |7dkr|hptjp}hd|cv rhtd |cd}|ɡ ʡ }|jːrB|j|cd< |j̐rL|j|cd< |j͐rV|j|cd< dy|cvsb|cdy du rh|jj|cdy< tРш }|dkr dd2 tjK ||h|f|c|b|5tL |d|?||[|\|0|$d}u|uW S |dkr dd2 tcjK |||[|0|f|c|bd||?tL |$|d|\dM}u|uW S tcjK |||[|0|f|c|bd||?tL |$|d|\dM}u|uW S |7dkrtjKdi d d|dG|?d|fdt^d0|$d7|0db|[d|dd|cd|bd=|5d|d|hdc|\dtL d?d}u|uW S |7dkr|$p<|cddp<tTdp<tTdp<tTd}$|0pf|cd|cd7|cd.dpftTdpftTdpftTdpftTd}0|cd|cdd}d}|dur|d|0}0|d|d0|$}$|d|dd}|dur||cd< tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?dd|dG|?dtL d0|$d|ddc|\}u|uW S |7dkr|hptjp}htjK ||h|ft^|c|b|5tL |dd
}fd|cv r|cd dnu rt|f d|dd"}u|uW S |f}u|uW S |7dkrtjRp.|0p.tQdp.d}0|$durCd|?vrCd|$ |?d< tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?dd|dG|?dtL d0|$d|ddc|\}u|uW S |7dkrtjRp|0ptQdpd}0|$ptjՐptjdptjV}$|$durd|?vrd|$ |?d< tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?dd|dG|?dtL d0|$d|ddc|\}u|uW S |7dk rtjR p|0}0tcjK |||[|0|f|c|b|'|7||?tL |$|dd}u|uW S |7dk rl|$ p1tj֐ p1tjV p1tQd}$tQd}|0 pKtjR pKtQd pKd| d}0|h pQtjp}htcjK |||[|0|f|c|b|'d||?tL |$|dd}u|uW S |7dk sx tjv  r|0 p}tjR}0d}7|cdd	}tjK ||0|ft^|c|b|5tL |d|\d}f|dnu  r|fd d
 d d }t| d|dd"}u|uW S |f}u|uW S |7dk sȈ tjv !r.zI|d	u  rt|dnd}\tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd5|'d|dc|\d?|7dtL d|}uW |uW S  t!y- }D z|dja||$t|DdG|?id |Dd}D~Dww |7dk!rTtjR!p9|0}0tcjK |||[|0|f|c|b|'d||?tL |$|dd}u|uW S |7dk!r|$!pitjې!pitTd!pitjV}$tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd|dc|\d?|7dtL d|dtܓ}u	 |uW S |7dk!r|$!ptjݐ!ptTd!ptjV}$tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd|dc|\d?|7dtL d|dtޓ}u	 |uW S |7dk"s tjv "r`|$"ptj"ptTd"ptjV}$|0"p#tjR"p#tTd"p#d}0tcjKdi d d|dG|?d|fd0|$d7|0db|[d|dd|cd|bd|dc|\d?|7dtL d|dt}u	 |uW S |7dk"rtjR"po|0"pod2}|du "s{|d2k"rt d	 dҠyddq |D }ztjj||? |zg|
|||)d¡dÜdĜ|)di dō}| }	 |d d
 d d
 }||fjd
 j_tt |f_ |f_|f}u|uW S |7tjv #rUd}tjD ]}|d |7k"r|d }"q|du "rt |7dʍt|[||dˍ}|?#ptjX#pi }?|di d d|dG|?d|fdt^d0|$d7|0db|[d|dd|cd|bd=|5d|d|hdc|\dtL }u|dnu #rRt|u |7|dd̍W S |uW S |7dk#rd
dlm} | j|0#pjtjR|$#potjVdύ\}0}$|?#pztjX}?tcjKdi d d|d|db|[d7|0d|fd|cd|bd5|'d?|7d|dG|?dtL d0|$d|ddc|\}u|uW S t |7dʍ t#y }D z
t |7|D|*|)dЍd}D~Dww (  a  
    Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly)
    Parameters:
        model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/
        messages (List): A list of message objects representing the conversation context (default is an empty list).

        OPTIONAL PARAMS
        functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list).
        function_call (str, optional): The name of the function to call within the conversation (default is an empty string).
        temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0).
        top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0).
        n (int, optional): The number of completions to generate (default is 1).
        stream (bool, optional): If True, return a streaming response (default is False).
        stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true.
        stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens.
        max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity).
        max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
        modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request.. You can use `["text", "audio"]`
        prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content.
        audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"]
        presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far.
        frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
        logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
        user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
        logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message
        top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used.
        metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc.
        api_base (str, optional): Base URL for the API (default is None).
        api_version (str, optional): API version (default is None).
        api_key (str, optional): API key (default is None).
        model_list (list, optional): List of api base, version, keys
        extra_headers (dict, optional): Additional headers to include in the request.

        LITELLM Specific Params
        mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
        custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock"
        max_retries (int, optional): The number of retries to attempt (default is 0).
    Returns:
        ModelResponse: A response object containing the generated completion and associated metadata.

    Note:
        - This function is used to perform completions() using the specified language model.
        - It supports various optional parameters for customizing the completion behavior.
        - If 'mock_response' is provided, a mock completion response is returned for testing or debugging.
    Nzmodel param not passed in.)r   )r  )r  )r%  )r  Z_skip_mcp_handlerFr   )acompletion_with_mcp)LiteLLM_Proxy_MCP_Handler)	ToolParamr   r   r   r   r   r   r   r   r   r   r%  r   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r	  r&  r
  r  r  r  r   r  r   r  r  r  r  r  r  r(  r*  r+  r  force_timeoutr   	logger_fnverboser  r  r>  metadatar  proxy_server_requestr  provider_specific_headerheadersensure_alternating_rolesuser_continue_messageassistant_continue_message"Failed to get proxy auth headers: r   r   cooldown_timecontext_window_fallback_dictr   
ssl_verifyinput_cost_per_tokenoutput_cost_per_tokeninput_cost_per_secondoutput_cost_per_secondinitial_prompt_valuerolesfinal_prompt_value	bos_token	eos_tokenpreset_cache_keyhf_model_namesupports_system_message
base_model(disable_add_transform_inline_image_blocktext_completionatext_completionr   r   zno-logr  r!  litellm_system_prompt)r   r  r  r  r   )r  r   r"  r#  )r   r   r   r  r!  r"  r#  T)r   Zsystem_promptZmerge_with_first_systemc                    s    g | ]}|d   kr|d qS )Z
model_namelitellm_paramsr   )ry  mr  r   r   rz  5  s    zcompletion.<locals>.<listcomp>deploymentsr{  r^  r   r  r(  r   r  )r  r  r  Zaws_region_nameregion_namerequest_timeout/r  r   r  model_file_id_mapping)r   Zmodel_idr  c                 S   s   g | ]}|j qS r   )r  )ry  providerr   r   r   rz    s    r   r  allowed_openai_paramsadditional_drop_params)r   Zpassed_paramsZspecial_paramsr  r  Zremove_sensitive_keysZadd_provider_specific_paramsprovider_configfunctions_unsupported_model)r   r   litellm_call_idmodel_alias_mapZcompletion_call_idno_logazure_ad_token_providerZlitellm_trace_idZlitellm_session_idcustom_prompt_dictZlitellm_metadatadrop_paramsZ"merge_reasoning_content_in_choicesZuse_litellm_proxyazure_ad_token	tenant_idZ	client_idZclient_secretZazure_usernameZazure_passwordZazure_scopeZlitellm_request_debugZtpmZrpmr   r  optional_paramsr  r  r)  )
r   r   r*  r+  r  r   r)  r  r  r   )r   r  r  r  r  )responses_api_bridge)r   r   r  r  r   r(  r   ru  r  r  r   r   r  encodingr   )Zazure_client_paramsZazure_clientZAZURE_API_TYPEAZURE_API_BASEAZURE_API_VERSIONAZURE_OPENAI_API_KEYAZURE_API_KEY
extra_bodyAZURE_AD_TOKENr  dynamic_paramsr  rP  r  r  ru  api_type)r  r  r(  )rq  r   r  Zadditional_args
azure_textz}api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable.Zdeepseekr  r  r  )AzureFoundryModelInfoZagents)AzureAIAgentsConfigz^Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var.)r   r   r(  r   r  ru  r  r  r   r   r   r  r  z^Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var.z/v1/messagesz
/anthropicrg   r  z	command-rr,  zft:babbage-002zft:davinci-002r1  OPENAI_BASE_URLOPENAI_API_BASEhttps://api.openai.com/v1OPENAI_API_KEYrv   c                 S      g | ]}|d  qS rv  r   ry  r[  r   r   r   rz        )r   r   r  rP  r   r  r(  r   r   ru  r  r  r  r   r.  fireworks_aiZherokuZragflowr  ZgroqZGROQ_API_BASEzhttps://api.groq.com/openai/v1ZGROQ_API_KEYZbedrock_mantleZBEDROCK_MANTLE_API_BASEZBEDROCK_MANTLE_API_KEYZa2a)r   r(  r   r  r  ZA2A_API_BASEzapi_base is required for A2A provider. Either provide api_base parameter, set A2A_API_BASE environment variable, or register the agent in the proxy with model='a2a/<agent-name>'.gigachatGIGACHAT_API_KEYGIGACHAT_CREDENTIALSsapZaiohttp_openaicometapiCOMETAPI_KEYCOMETAPI_API_BASEhttps://api.cometapi.com/v1minimaxZMINIMAX_API_KEYZMINIMAX_API_BASEzhttps://api.minimax.io/v1hosted_vllmHOSTED_VLLM_API_BASEZcustom_openaiZ	deepinfra
perplexity
nvidia_nimZcerebrasZbaseten	sambanova
volcengineZanyscaletogether_ainebiuswandbZclarifaizft:gpt-3.5-turboOPENAI_ORGANIZATIONgithub_copilot)Authenticator)get_copilot_default_headersZ)EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLERmistralMISTRAL_API_KEYZMISTRAL_API_BASEzhttps://api.mistral.ai/v1Z	replicateZREPLICATE_API_KEYZREPLICATE_API_TOKENZREPLICATE_API_BASEzhttps://api.replicate.com/v1)r   r   r(  r  rP  r  r  r  r  r   ru  r  r   r  Zanthropic_textZANTHROPIC_API_KEYZANTHROPIC_API_BASEZANTHROPIC_BASE_URLz%https://api.anthropic.com/v1/completeZ$LITELLM_ANTHROPIC_DISABLE_URL_SUFFIXz/v1/completezILITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix)r   r   r   r   r(  r  r  r  r  r  r   r  r  r   ru  rr  z%https://api.anthropic.com/v1/messageszILITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffixZ	nlp_cloudZNLP_CLOUD_API_KEYZNLP_CLOUD_API_BASEzhttps://api.nlpcloud.io/v1/gpu/)r   r   r(  r  rP  r  r  r  r  r   ru  )r  ru  r   ZALEPH_ALPHA_API_KEYZALEPHALPHA_API_KEYZALEPH_ALPHA_API_BASEz$https://api.aleph-alpha.com/complete)r   r   r(  r  rP  r  r  r  r  Zdefault_max_tokens_to_sampler   ru  cohere_chatcohereCOHERE_API_KEY
CO_API_KEYzCohere route: v2ZCOHERE_API_BASEzhttps://api.cohere.com/v2/chatzv2/r  zhttps://api.cohere.ai/v1/chatzModel: z, API Base: zProvider Config: ZmaritalkZMARITALK_API_KEYZMARITALK_API_BASEzhttps://chat.maritaca.ai/api)r   r   r(  r  rP  r  r  r  r  r   ru  r  r  Zamazon_novaZAMAZON_NOVA_API_KEYZAMAZON_NOVA_API_BASEzhttps://api.nova.amazon.com/v1)r   r   r(  r  rP  r  r  r  r  r   ru  r   r  r  huggingfaceZHF_TOKENHUGGINGFACE_API_KEYZociZcompactifaiZCOMPACTIFAI_API_KEYzhttps://api.compactif.ai/v1r   )r   r   r  r(  rP  r  r  r   r  r  ru  
databricksDATABRICKS_API_BASEDATABRICKS_API_KEY)r   r   r   r   r(  r  r  r  r  r   r  r  r   ru  r   Z	datarobot
openrouterOPENROUTER_API_BASEhttps://openrouter.ai/api/v1OPENROUTER_API_KEY
OR_API_KEYOR_SITE_URLhttps://litellm.aiOR_APP_NAMEliteLLMzHTTP-RefererzX-Titler   vercel_ai_gatewayVERCEL_AI_GATEWAY_API_BASEhttps://ai-gateway.vercel.sh/v1VERCEL_AI_GATEWAY_API_KEYZVERCEL_SITE_URLZVERCEL_APP_NAME)zhttp-refererzx-titleZtogethercomputerr   zPalm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=envertex_ai_betageminivertex_projectvertex_ai_projectVERTEXAI_PROJECTvertex_locationvertex_ai_locationVERTEXAI_LOCATIONvertex_credentialsvertex_ai_credentialsVERTEXAI_CREDENTIALSZPALM_API_KEYGEMINI_API_BASEgemini_api_keyr  VERTEXAI_API_BASE)r   r  )VertexAgentEngineConfig)r   r   r  rP  r  r  r  r  r4  r1  r7  ru  r   Z	predibasepredibase_tenant_idZPREDIBASE_TENANT_IDzMissing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=<MY-ID>)`) or in env - `PREDIBASE_TENANT_ID`.ZPREDIBASE_API_BASEZPREDIBASE_API_KEY)r   r   r  rP  r  r  r  r  ru  r   r(  r  r   r  r   r-  z/https://codestral.mistral.ai/v1/fim/completionsZCODESTRAL_API_KEY)r   )r   r   r  rP  r  r  r  r  ru  r   r(  r  r   r   Zsagemaker_chat	sagemaker)r   r   r  rP  r  r  r  r  r  r  ru  r   r  aws_bedrock_clientz'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication.Zaws_access_key_idZaws_secret_access_keyZaws_session_tokenZconversez	converse/)r   r   r  r  r  r  r  r  ru  r  r   r   r   r(  r   Zconverse_likezconverse_like/watsonxZwatsonx_textZapikeyZWATSONX_APIKEYZWATSONX_API_KEYZ
WX_API_KEYrZ  ZWATSONX_API_BASEZWATSONX_URLZWX_URLZWML_URLwx_credentialsZwatsonx_credentialstokenZwatsonx_tokenZvllm)
r   r   r  r  rP  r  r  r  r  ru  ollamaOLLAMA_API_BASEhttp://localhost:11434AuthorizationzBearer Zollama_chatZOLLAMA_API_KEYtritonZ
cloudflareZCLOUDFLARE_API_KEYZCLOUDFLARE_ACCOUNT_IDZCLOUDFLARE_API_BASEz.https://api.cloudflare.com/client/v4/accounts/z/ai/run/Zpetals)r   r   r(  r  rP  r  r  r  r  ru  r   r  r[  	snowflakero  Zgradient_aiZbytezZBYTEZ_API_KEYZlemonadeZLEMONADE_API_KEYovhcloudOVHCLOUD_API_KEYOVHCLOUD_API_BASE0https://oai.endpoints.kepler.ai.cloud.ovh.net/v1ZcustomzGapi_base not set. Set api_base or litellm.api_base for custom endpointsc                 S   r  r  r   r  r   r   r   rz    r  top_k)promptr   r   r   rN  )r   r   )r  jsondataoutputr  custom_handlerr  )Zasync_fnr   Z
custom_llmrs  Z	langgraph)LangGraphConfig)r(  r   r2  r   )
ValueErrorrb   rc   re   rf   rd   r   r   Z.litellm.responses.mcp.chat_completions_handlerr  Z/litellm.responses.mcp.litellm_proxy_mcp_handlerr  Zlitellm.types.llms.openair  r   r   r   Z_should_use_litellm_mcp_gatewayr   boolr   updater   
proxy_authget_auth_headersr@  rh   warningr   rF  rp   rT   r9  r:  r;  Zget_chat_completion_promptro   r?  rn   Zbatch_completion_modelsr  rE   r  rK   rS   r  r  r)   Zget_provider_specific_headershasattrr  rb  ri  r`   readr   register_modelr  rq   r   r   rF   Zget_provider_chat_configZ'translate_developer_role_to_system_rolert   r   r]   Zadd_function_to_promptrs   r   update_environment_variablesr  r  Zlitellm.completion_extrasr  r   _get_encodingr1  ZAzureOpenAIZAsyncAzureOpenAIr|   rZ   r(  r  r;   AZURE_DEFAULT_API_VERSIONr   	azure_keyr  ZAzureOpenAIO1ConfigZis_o_series_modelZ
get_configr   azure_o1_chat_completionsrP  ZAzureOpenAIConfigazure_chat_completionsr  azure_text_completionsbase_llm_http_handlerZ"litellm.llms.azure_ai.common_utilsr  Zget_azure_ai_routeZlitellm.llms.azure_ai.agentsr  Zget_api_baserR   r  rstripendswithsplit azure_anthropic_chat_completionsr  rw   Z+openai_text_completion_compatible_providersr  
openai_keyrH  r   r<  listjoinopenai_text_completionsrI  Zgroq_keyZGroqChatConfigZBedrockMantleChatConfigZ	A2AConfigZ"resolve_agent_config_from_registrygigachat_keyZGenAIHubOrchestrationConfigsap_gen_ai_hub_chat_completionsbase_llm_aiohttp_handlercometapi_keyopen_ai_chat_completion_modelsopenai_compatible_providersr6   existsZ)litellm.llms.github_copilot.authenticatorr  Z(litellm.llms.github_copilot.common_utilsr  Zenable_preview_featuresrY   ZOpenAIConfigr:   openai_chat_completionsZreplicate_modelsreplicate_keyreplicate_chat_completionZclarifai_modelsZanthropic_keyosenvironr=  anthropic_chat_completionsnlp_cloud_keynlp_cloud_chat_completionrB   aleph_alpha_keyr   r   
cohere_keyr3   Zget_cohere_router  maritalk_keyopenai_like_chat_completionZamazon_nova_api_keyhuggingface_keyr   getenvdatabricks_keyopenrouter_keyZOpenrouterConfigZVercelAIGatewayConfigZtogether_ai_modelsr1  r4  r   rl   vertex_chat_completionr8   r7   ZPARTNER_MODELS%vertex_partner_models_chat_completionZGEMINIZGEMMAvertex_gemma_chat_completionZMODEL_GARDEN#vertex_model_garden_chat_completionZAGENT_ENGINEZ2litellm.llms.vertex_ai.agent_engine.transformationr=  r   r>  Zpredibase_keypredibase_chat_completionsrH   codestral_text_completionssagemaker_llmZ_get_credentialsZget_frozen_credentialsZ
access_keyZ
secret_keyrC  metar  r2   Zget_bedrock_route bedrock_converse_chat_completionwatsonx_chat_completionvllm_handlerZ
ollama_keyZcloudflare_api_keyZpetals_modelspetals_handlerZsnowflake_modelsr5   Z	bytez_keybytez_transformationZlemonade_keylemonade_transformationZovhcloud_modelsZovhcloud_keyovhcloud_transformationZmodule_level_clientpostrP  r  r[  rv  r   rm  r  r   _custom_providerscustom_provider_mapr#   r   Z*litellm.llms.langgraph.chat.transformationrT  Z$_get_openai_compatible_provider_infor   )r   r   r   r   r   r   r   r   r%  r   r   r   r   r   r   r   r  r  r
  r  r  r  r  r  r  r	  r  r  r&  r  r  r  r   r   r   r  r   r  r  r  r  r   rQ  Zskip_mcp_handlerr  r  r  Ztools_for_mcpr(  r*  r+  r  r  r  r  r  r  r>  r  r  r  r  r  r  r  r  r  proxy_headersrO  r   r   r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r  r  r!  r  r   r  r  r  r  r  dynamic_api_keyr  r  Zoptional_param_argsZprocessed_non_default_paramsr  r  r  r  r  r  configr   r   r   r  Zazure_ai_router  partsrO  	_responser  r  Zcopilot_authZcopilot_api_keyZcopilot_headersZopenai_metadataZuse_base_llm_http_handlerrv  Zdisable_url_suffixr{  r}  r~  Zcohere_router  r  Z
hf_headersopenrouter_site_urlopenrouter_app_nameopenrouter_headers_headersrQ  Zvercel_site_urlZvercel_app_nameZvercel_headersr2  r5  r7  r;  
new_paramsZmodel_router=  Zvertex_agent_engine_configr  Z_model_responseZtext_completion_model_responser@  credsZbedrock_routerB  rC  Z
account_idZresp_stringrZ  respZresponse_jsonZstring_responserS  item
handler_fnrT  r   r  r   r     s,'  e





	
 !"#$%&'(),




	




















	
 !"#
$&
	







 !"#&'
(
)
*
+
,
-
./01
2
3
5









	


	
                    _                    





	
                                        
	
                                       







                   R                   




	
                                      




	

                  b                  







                                    
	
                 }                 
		
                 _                 
		
                 @                 
	
                 "                 
		

	
                k                


	
                Q                

	
                &                
	
                               



	
               ]               
	
               6               
	
                            
	
              _              
	
              D              


















	
	

             5             	
             %             	            a                        [            
	

            =            
	

	
           r                      C           	                      



	
          q                    @                    2                                                 	
         m                  C         

                  	
         
        




	
        7        




	
       w       
       l       	
       K       	
	
	
	



	


            


	

     Q     



     *                        

    b        O        =    	
    +    		



	
   i      ;   	
   0   	
       v  
  U  
  '  	
  !  	   	
 U 	
 9 	
  

	W



	


,

	
r   c               
   O   s   zddl }W n ty } ztd| d}~ww |dd}d|d< d|d< |dd}|d	t}|d
krI|j|jddd||dd}n
|j||dd}||g| R i |S )z8
    Executes a litellm.completion() with 3 retries
    r   N?tenacity import failed please run `pip install tenacity`. Errorr      r   retry_strategyconstant_retryoriginal_functionexponential_backoff_retryrg   
   
multipliermaxTr   r%  reraiser%  r  )tenacityr@  r   r   Retryingwait_exponentialstop_after_attemptrQ  r   r  rO  r   r  r  retryerr   r   r   completion_with_retries  s2   
r  c               
      s   zddl }W n ty } ztd| d}~ww |dd}d|d< d|d< |dd}|d	t}|d
krJ|j|jddd||dd}n
|j||dd}||g| R i |I dH S )z{
    [DEPRECATED]. Use 'acompletion' or router.acompletion instead!
    Executes a litellm.completion() with 3 retries
    r   Nr  r   r  r   r  r  r  r  rg   r  r  Tr  r  )r  r@  r   r   AsyncRetryingr  r  r  r   r   r   acompletion_with_retries9  s0   
r  c            	   
   O   s   zddl }W n ty } ztd| d}~ww ddlm} |dd}d|d< d|d< |dd	}|d
|}|dkrO|j|jddd||dd}n
|j||dd}||g| R i |S )z5
    Executes a litellm.responses() with retries
    r   Nr  )r  r   r  r   r  r  r  r  rg   r  r  Tr  r  )r  r@  litellm.responses.mainr  r   r  r  r  )	rQ  r   r  rO  r  r   r  r  r  r   r   r   responses_with_retriesW  s4   
r  c            	   
      s   zddl }W n ty } ztd| d}~ww ddlm} |dd}d|d< d|d< |dd	}|d
|}|dkrP|j|jddd||dd}n
|j||dd}||g| R i |I dH S )z6
    Executes a litellm.aresponses() with retries
    r   Nr  )
aresponsesr   r  r   r  r  r  r  rg   r  r  Tr  r  )r  r@  r  r  r   r  r  r  )	rQ  r   r  rO  r  r   r  r  r  r   r   r   aresponses_with_retriesy  s2   
r  c               
      sF  t  }t| dkr| d n|d }d|d< |dd}zlttg| R i |}t }t|j|}t	|||ddd\}}}}|
d|I dH }	d}
t|	trZtdi |	}
nt|	trb|	}
n
t |	rl|	I dH }
|
durt|
trt|
d	r||
jd< |
du rtd
|
W S  ty } z|pd}t|||| |dd}~ww )ay  
    Asynchronously calls the `embedding` function with the given arguments and keyword arguments.

    Parameters:
    - `args` (tuple): Positional arguments to be passed to the `embedding` function.
    - `kwargs` (dict): Keyword arguments to be passed to the `embedding` function.

    Returns:
    - `response` (Any): The response returned by the `embedding` function.
    r   r   T
aembeddingr  Nr(  r'  r  zCUnable to get Embedding Response. Please pass a valid llm_provider.r1  r2  r   )r7  r8  r<  r   r   	embeddingrB  rC  rD  rS   rE  r9  rF  rC   rG  r[  r  rU  r@  r   rQ  r   r0  r   r  rK  rL  rM  rJ  rN  r   rO  r   r   r   r    sX   





r  r   
dimensionsencoding_formatr(  r  cachingTc                K      d S r   r   r   rq  r  r  r   r(  r  r   r  r  r  r  r  r  r  r   r   r   r   r       r  )r  c                K   r  r   r   r  r   r   r   r    r  c           8         sP  | dd}|dd}| dd}| dd}| d}| dd}| dd}| d	d}| d
d}| ddp<|}|du rCi }|durL|| tjdurxztj }|| W n tyw } ztd|  W Y d}~nd}~ww | dd}| dd}| dd}g d}d	d
gt	 }||   fdd|
 D } t| |||d\} }}!}|!dur|!}| dd}"td| |
||||"d| }#|dur|dus|durt| d|  t||| ddi tdi |}$|}%|%j| |
|#|$|d |durt| |dS z d}&|du s|dkr|ptjptd}|p.tjp.tdp.tj}|#ddp9td}'|pHtjpHtjpHtd }|du rRtd!tjdi d"| d#|d$|d%|d&|d|'d|d'|%d(|d)t d*|#d|d	|d|d|p|d+|$}&nW|d,kr|ptj}tj| |||||%|t |#|||$d-}&n8|d.ks|d/ks|d0ks|d1ks| tjv r,|du r,|ptjptd2ptd3pd4}tjptd5pdt_|ptjptj ptd6}|dur|i kr||#d
< |dur||#d7< nd|#d7< d}t!j| ||||%|t |#||||d8}&n|d9kr]|p;tjp;t"d:}|pJtjpJtj#pJt"d;}t$j| ||||%|t |#||d<
}&n|d=kr|pltjpltd>}|du rztjpytd?}tj| |||||%|t |#|||$|pi d@}&nS|dAks|dBks|dCkr|ptjptdD}|du r|ptjptj%ptdE}|dur|i kr||#d
< t&j| ||||%|t |#||d<
}&n|dFks|dGkr|ptj'ptdHptdIptj}(|du ri }tj| ||||(|%|t |#|||$|d@}&n|dJkry|p,tjp,tdKp,dL}|p@tjp@tj(p@tdMp@tdN}t"dOpGdP})t"dQpNdR}*|)|*dS}+|pYtj)},|,rb|+|, |+}tj| |||||%|t |#|||$|d@}&nl|dTkr|ptjptdUpdV}|ptjptdWptdX}tj| |||||%|t |#|||$|d@}&n4|dYkr|ptj*pt"dZptj}t+j| |t, |||%t |#|||$d[}&n|d\krt-|t.r|g}-n|}-t/j0| |-t, |%|#t |||i |t1||d]}&n|d^kr#|du rtd_tj| |||||%|t |#||i d-}&n|d`krw|p1t2 p1tj}.|p<tjp<tda}t3j4di d"| d#|dbt, d'|%d*|#d)t dcdddddedd	|dft1dgd`d%|.d$|d|d
|}&nn|dhkr;|#dcdp|#didptj5ptdjptdk}/|#dddp|#dldptj6ptdmptdn}0|#dedp|#dodptdpptdq}1|ptjptdrptds}dt|#v sdu|#v s| t7j8v rt7j9| |t, |%|#|$t |/|0|1|t1dh||dv}&nt:jdi d"| d#|dbt, d'|%d*|#d)t dc|/dd|0de|1dgdhd(|d	|dft1d%|d$|d|}&n|dwkrQt;j| |t, ||%|#t |dx}&n|dykrtjpc|pctdzpcd{}t-|t.rm|g}t<d|d} |D stj=d~| | dyd|du rt>j?nt>j@}2|2|| |t, |%|#t d}&nI|dkrtAj| |t, |%|#t t1d}&n4|dkr|ptjptd}t!j| ||||%|t |#||d<
}&n|dkr|ptjptd}t!j| ||||%|t |#||d<
}&n|dkr%|ptjptd}|ptjptdpd}t!j| ||||%|t |#||d<
}&n|dkrU|p4tjp4td}|pBtjpBtdpBd}t!j| ||||%|t |#||d<
}&n|dkr|pdtjpdtd}|prtjprtdprd}tj| |||||%|t |#||i d-}&n^|dkrtj| |||||%|t |#||i d-}&nE|dkrtj| |||||%|t |#||i d-}&n,|dkrtBjC|#||d}3|3d% }|3d$ }d|3v r|3d |#d< tj| |||||%|t |#i ||d}&n|dkr |ptjptdpd}|ptjptdpd}t!j| ||||%|t |#||d<
}&n|dkr9tj| |||||%|t |#i ||d}&n|dkrj|pHtjpHtd}|pWtjpWtj pWtd}tDj| ||||%|t |#||d<
}&n{|dkrt-|t.ry|g}-n|}-tj| |-||||%|t |#i ||d}&nV|dkr|ptjptdptd}4|4du rtd|durt-|tEr|}ni }tj| ||||%||#i t |4|||d}&n|dkr|ptjptd}|ptjptdpd}tj| |||||%|t |#||i d-}&n|dkr8|ptjFptdptj}|p$tjp$tdp$d}tj| |||||%|t |#||i d-}&n|tjGv rvd}5tjHD ]}6|6d |krP|6d }5qC|5du r]tI| |d|sc|5jn|5jJ}7|7| ||%||||#t t1|$d
}&no|dkr|ptd}tj| |||||%|t |#||i d-}&nP|dkr|ptjptjKptdptd}tj| |||||%|t |#||d| ddid-}&n|dkrtj| |||||%|t |#||i d-}&ntI| |d|&durtL|&drt-|&tr||&jMdg< |&du 	rtI| |d|&W S  t	y' } z|jN||t.|d tO| |||dd}~ww )aw  
    Embedding function that calls an API to generate embeddings for the given input.

    Parameters:
    - model: The embedding model to use.
    - input: The input for which embeddings are to be generated.
    - encoding_format: Optional[str] The format to return the embeddings in. Can be either `float` or `base64`
    - dimensions: The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.
    - timeout: The timeout value for the API call, default 10 mins
    - litellm_call_id: The call ID for litellm logging.
    - litellm_logging_obj: The litellm logging object.
    - logger_fn: The logger function.
    - api_base: Optional. The base URL for the API.
    - api_version: Optional. The version of the API.
    - api_key: Optional. The API key to use.
    - api_type: Optional. The type of the API.
    - caching: A boolean indicating whether to enable caching.
    - custom_llm_provider: The custom llm provider.

    Returns:
    - response: The response received from the API call.

    Raises:
    - exception_type: If an exception occurs during the API call.
    r^  Nr   r  r   r  r*  r  r  r  r  r  r  r  r  )r  r  r  r(  r  r   r&  r   r   r   r   r   r  c                    s   i | ]\}}| vr||qS r   r   )ry  r   r   Zdefault_paramsr   r   
<dictcomp>l  s    zembedding.<locals>.<dictcomp>r  r  )r   r  r  r  r  r  r  r  r  r  )r   r*  Tr  r  r  r  r  zPNo API Base provided for Azure OpenAI LLM provider. Set 'AZURE_API_BASE' in .envr   rq  r(  r   r  ru  r   r  r  r  r  )r   rq  r  r(  r   ru  r   r  r  r   r  r  r1  r  r  Zlitellm_proxyr  r  r  r  r  r  )r   rq  r(  r   ru  r   r  r  r   r  r   r  r  r  r   )
r   rq  r(  r   ru  r   r  r  r   r  r  r	  ZHOSTED_VLLM_API_KEY)r   rq  r  r(  r   ru  r   r  r  r   r  r  r  Zopenai_likeZ	llamafileZ	lm_studioZOPENAI_LIKE_API_BASEZOPENAI_LIKE_API_KEYr  r  r  r  r!  r"  r#  r$  r%  r&  r'  r(  r)  r*  r+  r,  r-  r.  ZVERCEL_OIDC_TOKENr  r  )r   rq  r  r   r(  ru  r  r  r   r  r  r  )r   rq  r  ru  r  r  r   r   r  r  r(  rP  r  r   rH  z7api_base is required for triton. Please pass `api_base`r0  r:  r  r1  r4  r7  rP  r  r  r2  r3  ZVERTEX_PROJECTr5  r6  ZVERTEX_LOCATIONr8  r9  ZVERTEX_CREDENTIALSr<  ZVERTEX_API_BASEimageZvideo)r   rq  r  ru  r  r  r  r1  r4  r7  r  rP  r  r   r(  r   )r   rq  r  r(  ru  r  r  r   rD  rE  rF  c                 s   s    | ]}t |tV  qd S r   )r9  r   )ry  r  r   r   r   	<genexpr>  s    zembedding.<locals>.<genexpr>z+Invalid input for ollama embeddings. input=r[  r   rW  )r(  r   Zpromptsr  ru  r  r  r?  )r   rq  r  ru  r  r  rP  r  r  r  ZFIREWORKS_AI_API_KEYr  ZNEBIUS_API_KEYZNEBIUS_API_BASEzapi.studio.nebius.ai/v1r  ZWANDB_API_KEYZWANDB_API_BASEz!https://api.inference.wandb.ai/v1r  ZSAMBANOVA_API_KEYZSAMBANOVA_API_BASEzhttps://api.sambanova.ai/v1ZvoyageinfinityrA  )r  r   r(  rC  )r   rq  r  r(  r   ru  r   r  r  r  r   r  Z
xinferenceZXINFERENCE_API_KEYzstub-xinference-keyZXINFERENCE_API_BASEzhttp://127.0.0.1:9997/v1r  r  ZAZURE_AI_API_BASEZAZURE_AI_API_KEYZjina_air  ZARK_API_KEYZVOLCENGINE_API_KEYzuMissing API key for Volcengine. Set ARK_API_KEY or VOLCENGINE_API_KEY environment variable or pass api_key parameter.)r   rq  r   r  ru  r(  r  r  r  r   r   r  r  rJ  rK  rL  rM  r  r  r  r  r  rS  r  )
r   rq  ru  r(  r   r   r  r  rP  r  rI  ZSNOWFLAKE_JWTr  r  r   r  r
  r  r  )r   r3  r  r5  r   )Pr   r   rW  r   rX  rY  r@  rh   rZ  r   r   rS   rV   r]  r  r   r^  r,   r(  r;   r  r`  r   ra  rU  rc  r  rC   re  Zopen_ai_embedding_modelsr   r1  rj  ru  rZ   r  databricks_embeddingZopenai_like_keyopenai_like_embeddingr~  r  r  r  huggingface_embedr_  r9  r   bedrock_embeddingZ
embeddingsrP  r   google_batch_embeddingsZbatch_embeddingsr1  r4  vertex_multimodal_embeddingZ%SUPPORTED_MULTIMODAL_EMBEDDING_MODELSZmultimodal_embeddingvertex_embeddingr   allBadRequestErrorrD  Zollama_aembeddingsZollama_embeddingsr  r   Zget_watsonx_credentialsazure_ai_embeddingrF  rq  r  r  r#   r  rn  r[  r  r  r   )8r   rq  r  r  r   r(  r  r   r  r  r  r  r  r  r   r^  r   r  r   r  r*  r  r  r  r  r  rO  r  r  r  Zopenai_paramsr  r   r  r  r  litellm_params_dictr  r   r  r~  r  r  r  r  Ztransformed_inputr;  r2  r5  r7  Zollama_embeddings_fncredentialsZvolcengine_keyrS  r  r  r   r  r   r    s   -








	





































	








	







	

	





































c               
      s|  t  }t| dkr| d n|d }d|d< d}zttg| R i |}t }t|j|}|d|I dH }t	|t
sBt	|trRt	|t
rOtdi |}	n|}	nt |r]|I dH }	n|}	|dddu sqt	|	tsqt	|	trtt|	||| d|||d	d
W S t	|	tr|	W S t |	r|	I dH }	t }
tjjj|
|	|d}
|
W S  ty } z|pd}t|||| |dd}~ww )zP
    Implemented to handle async streaming for the text completion endpoint
    r   r   Tr   Nr   F)r   r   r  rQ  r   )rt  r   r  r   )text_completion_responser   r  r1  r2  r   )r7  r8  r<  r   r  rB  rC  rD  rE  r9  rF  rH   rG  r   rI   rB   rS  r   r  LiteLLMResponseObjectHandlerconvert_chat_to_text_completionr@  r   )rQ  r   r0  r   r  rK  rL  rM  rN  r   r  rO  r   r   r   r  T  sr   




r  rO  best_ofechor%  suffixc           %         sr  dd l }	 dv rd }d u rt|tr|d t i |d ur*|d< |d ur2|d< |d ur:|d< |d urB|d< |d urJ|d< |d urR|d< |d urZ|d	< |	d urb|	d
< |
d urj|
d< |d urr|d< |d urz|d< |d ur|d< |d ur|d< |d ur|d< |d ur|d< |d ur|d< |d ur|d< |d ur|d< |d ur|d< t||d\}}}}|dkrB|du rdvrdd< t| trBdd l}t	dt
| dkrBt| d trBdd | D } fdd|j ,fd dt| D }t|j|D ]\}} |  ||< q |_W d    S 1 s;w   Y  S g }!t| trjt
| dkrjt| d trj| D ]}"d|"d!}#|!|# q[nRt| trwd| d!g}!nE|d"ks|d#ks|d$ks|d%ks|d&krt| trt
| dkrt| d tst| d trd| d!g}!ntd'|  d(d)d  |d ur|d"kr|tjvrd*| dd  d u rtd+dd,< t |!d-}$d.d/du r |$S |du sdd/du st|$trt|$||d0}$|$S t|$tr&|$S t|$tr.|$S tjjj|$d1S )2Nr   Zenginer  r  r   r  r  r   r   r   r%  r   r   r  r   r   r  r(  r  r   r  r'  r  TZtop_n_tokensr  ztext-davinci-003c                 S   s   g | ]}d qS r   r   )ry  xr   r   r   rz  6  s    z#text_completion.<locals>.<listcomp>c                    sp    |}i }t |dd|}|dd d< dd< |dd d< |dd d< |d d	 S )
Nr  )r   rO  r   r>  r  objectr  r   r  r   )decoder  r   )r  individual_promptZdecoded_prompt
all_paramsr   )rQ  r   r   r  r  	tokenizerr   r   process_prompt8  s   
z'text_completion.<locals>.process_promptc                    s   g | ]\}}  ||qS r   )submit)ry  r  r  )executorr  r   r   rz  J  s    )rw  rv  r1  r^  r  r-  r,  zVUnmapped prompt format. Your prompt is neither a list of strings nor a string. prompt=z;. File an issue - https://github.com/BerriAI/litellm/issuesrO  ztext-completion-openai/z;model is not set. Set either via 'model' or 'engine' param.r  r   r   F)rt  r   r   r  )r   r  )r  r9  r   r   rH   rS   rk  concurrent.futurestiktokenZencoding_for_modelr<  r   r   	enumerateas_completedresultr  r  r   r@  r   rr  rU  r   r   rB   rI   r  r  r  )%rO  r   r  r  r   r  r  r   r   r   r%  r   r   r  r   r   r  r(  r  r   r  r  rQ  r   r  Z_engine_modelr  
concurrentr  Zcompleted_futuresr  futurer   pr[  r   r   )rQ  r  r   r   r  r  r  r  r   r    s  5




*










r  
adapter_idc              
      s   zJd}t jD ]}|d | kr|d }q|du r!td| t j|j|d}tdi |I dH }d}t|tr>|j|d}t|t	rI|j
|d}|W S  tyY } z|d}~ww )	zD
    Implemented to handle async calls for adapter_completion()
    Nr>  adapterHNo matching adapter given. Received 'adapter_id'={}, litellm.adapters={}r  r   rt  r   )r   adaptersrU  r  !translate_completion_input_paramsr   r9  rE   "translate_completion_output_paramsrB   ,translate_completion_output_params_streamingr@  )r  r   translation_objr  
new_kwargsr   translated_responserO  r   r   r   aadapter_completion  s>   


r  c                     sT   ddl m} ttttttttf tt	 f f |j
di | ddi}|I d H S )Nr   )"GenerateContentToCompletionHandler	_is_asyncTr   )Z%litellm.google_genai.adapters.handlerr  r   r   r	   r   r   r   r
   bytesZgenerate_content_handler)r   r  coror   r   r   aadapter_generate_content  s    
r  c                 K   s   d }t jD ]}|d | kr|d }q|d u rtd| t j|j|d}tdi |}d }t|tr;|j|d}|S t|t	sEt
|rK|j|d}|S )Nr>  r  r  r  r  r  r   )r   r  rU  r  r  r   r9  rE   r  rB   inspectisgeneratorr  )r  r   r  r  r  r   r  r   r   r   adapter_completion  s4   


r  rq  c                 K   s   |pt jpt jptd}|dd }|dd }|d u r.|d ur(tj||d}ntj|d}|d ur;|jj| |d}n|jj| d}|	 }t j
jj|dS )	Nr  r(  r   )r   r   )r   rq  r   rq  r/  )r   r   rj  r;   r   r1  ZOpenAImoderationsr   
model_dumpr  r  convert_to_moderation_response)rq  r   r   r   r(  openai_clientr   response_dictr   r   r   
moderation  s(   r  c                    s<  ddl m} |ptjptjptd}tdi |}|dd }d }ztj|p(d||j	|jd\}}}	}W n
 tj
y?   Y nw |dd }
|
d u sOt|
|s[tjd||j	pW|d	}n|
}|pctjjj}|d urx|j||d
d i i ||d |d ur|jj| |dI d H }n
|jj| dI d H }| }tjjj|dS )Nr   )AsyncOpenAIr  r  r  r  r   T)is_asyncr   r(  r  r  r  r  r	  r   )r1  r  r   r   rj  r;   r<   r   rS   r(  r  r9  ru  Z_get_openai_clientr   ZOPENAIr  r^  r
  r   r  r  r  r  )rq  r   r   r  r   r  r  r  Z_dynamic_api_baseZ_dynamic_api_keyr  Z_openai_clientr   r  r   r   r   amoderation'  sn   

r  c               
      s  t  }t| dkr| d n|d }d|d< |dd}d}zttg| R i |}t }t|j|}t	||ddd\}	}}	}	|
d|I dH }
t|
trYtdi |
}nt|
tra|
}nt |
rl|
I dH }n	|
d|I dH }t|tstd	t| |durt|ts|durt|d
d}|du rt|}|dur||jd< |W S  ty } z|pd}t|||| |dd}~ww )a
    Calls openai + azure whisper endpoints.

    Allows router to load balance between them
    r   r   TatranscriptionfileNr(  r   r(  zVInvalid response from transcription provider, expected TranscriptionResponse, but got durationaudio_transcription_durationr1  r2  r   )r7  r8  r<  r   r   transcriptionrB  rC  rD  rS   rE  r9  rF  rJ   rG  rU  typer   ra  r&   r  r@  r   )rQ  r   r0  r   r  r  rK  rL  rM  rJ  rN  r   existing_durationcalculated_durationrO  r   r   r   r  s  s\   



	
r  r  language)rP  rV  ZsrtZverbose_jsonZvtttimestamp_granularities)wordsegmentr   c           "      K   sB  | dd}| dd}| dd}| dd}|dd}| d}| d	d}| d
d}|dg  t|}|dd}|rHt||jd< |du rOtj}tj	 }t
| ||
|	d\} }}}
|durf|}	td0| ||||||d|}td0i |}|j| |i ||||di d||d d}tj| t|d}|dkr|
ptjptd}
|ptjptd}|ddptd}|	ptjptjptd}	||d	< tj| ||||||||
|	||||d}n|dks|tjv r$|
ptjptdptdpd}
tjptdpdt_|	ptjptjptd}	tj| |||||||||
|	|||d}nN|durrtjd0i d | d!|d"|d#|d$|d|d|durSt|tsQt|t rS|ndd%|d&|d'|d(|
d)|	d*|d+i d,|d
|}|durt|t!st"|d-d} | du rt#|}!|!dur|!|j$d.< |du rt%d/|S )1r  r  Nr  r  r  r  Fr  r  r  tagsr   r  )r   r  rO  r  r  r   r  r  r  r  r  r  Zstream_responser  r  r^  r  r  r  r  r  )r   
audio_filer  r  r  r   r   ru  r(  r   r  r  r   r  r1  r  r  r  r  r  )r   r#  r  r  r  r   r   ru  r   r(  r   r  r  r  r   r#  r  r  r  r   r   ru  r(  r   r  r  r  r  r  z8Unmapped provider passed in. Unable to get the response.r   )&r   r   rU   r   model_call_detailsr1  DEFAULT_MAX_RETRIESr   r  rJ   rS   rX   r   r^  rF   Z'get_provider_audio_transcription_configr   r(  r;   r  r   ra  azure_audio_transcriptionsZaudio_transcriptionsrs  rZ   r   rj  openai_audio_transcriptionsre  r9  r5   r4   r   ra  r&   r  rU  )"r   r  r  rO  r  r  r   r  r   r   r(  r  r   r  r   r  r  r  r  r  r  r  r  r   r   r  r  r  r  r   r  r  r  r  r   r   r   r    sX  

		
	


	



r  c               
      s   t  }t| dkr| d n|d }d|d< |dd}zEttg| R i |}t }t|j|}t	||ddd\}}}}|
d|I dH }	t |	rW|	I dH }
|
W S |
d|I dH }
|
W S  ty{ } z|pmd	}t|||| |d
d}~ww )z%
    Calls openai tts endpoints.
    r   r   Taspeechr  Nr(  r  r1  r2  )r7  r8  r<  r   r   speechrB  rC  rD  rS   rE  rG  r@  r   r  r   r   r   r(    s:   

r(  voicer   projectr  speedinstructionsr  c           2      K   s  | dd }| dd }| dd }| dd }| dd }| dd }t| ||d\} }}}|dg  i }|d ur?||d	< |d urG||d
< |d urO||d< |
d u rVtj}
|d u r`tjp_tj}tdEi |}t	j
| t|d}|d ur|j| ||d|d\}}tt| d}|j| ||||||	d i d||d d }|dks|tjv r|d u st|tstjd| |d|ptjptdptdpd}|ptjptjptd}|ptjptdpd }|ptjptdpd }|ptj}tj| |||||||||
|||d}nj|dkr| dr@ddlm} |d u r!tjd | |dt||} | j dE| ||||||
|t!|p6d||d!|}n-|d u sKt|tsStjd"| |d|p]tjp]td#}|phtj"phtd$}|p|tjp|tj#p|td%p|td&}| d'i d(d ptd)}!| d*d }"|r||d< t$j| |||||||!|"|||
|||d+}n|d,kr&dd-l%m&}# |d u r|# }t|#|}$t|tr|nd }%|%d u s|%' stjd.| |d|%' }%||#j(d }&t|&t)r|&||#j(< |%||#j*< |d ur||d/< |d ur||d0< t!j+| ||%|$|||||
|||p!dd1}nG|d2ks0|d3krdd4l,m-}' t.dEi |}(d5| v rXd6d7l/m0}) |)j1| |||||pSi ||d8S |d u r`|' }t|'|}*|2|(j3|(j4|(j5d9 |*j dE| ||||||
|t!|pd|(jd d!|}n|d5krd6d7l/m0}) |)j1| |||||pi ||d8S |d:krdd;l6m7}+ |d u rtjd<| |dt|+|},|,j dE| ||||||
|t!|pd||d!|}n|d=kr=dd>l8m9}- |d u r|- }t|-|}.|d ur||d/< |d ur||d0< d }/t|tr|}/nt|t)r(| d?p'| d@p'| dA}/t!j+| ||/|.|||||
|||p9dd1}n0|dBkrmddCl:m;}0 |d u rP|0 }t|0|}1|1j dE| ||||||
|t!|ped||d!|}|d u r{t<dD=|tj>|S )FNr  r  r  r  r  r  r'  r!  r  r,  r-  r  F)r   r  r*  r  r   r  r"  r  r1  z;'voice' is required to be passed as a string for OpenAI TTSr  r  r  r  r  r  ZOPENAI_PROJECT)r   rq  r*  r  r   r(  r   r+  r   r   r   r(  r  r^  zspeech/r   )AzureAVATextToSpeechConfigz,Azure Speech Service configuration not found)r   rq  r*  r  r  ru  r   r  re  r(  r(  r   z:'voice' is required to be passed as a string for Azure TTSr  r  r  r  r  r  r  r  )r   rq  r*  r  r   r(  r  r  r  r   r   r   r   r(  r  Z
elevenlabs)ElevenLabsTextToSpeechConfigzA'voice' must resolve to an ElevenLabs voice id for ElevenLabs TTSr(  r   )r   rq  r*  text_to_speech_provider_configZtext_to_speech_optional_paramsr  r  ru  r   r  r   r   r  r/  )VertexAITextToSpeechConfigr0  rg   )#speech_to_completion_bridge_handler)r   rq  r*  r  r  r  ru  r  )r1  r4  r7  Zrunwayml)RunwayMLTextToSpeechConfigz/RunwayML Text-to-Speech configuration not foundr  )MinimaxTextToSpeechConfigvoice_idr>  nameZ	aws_polly)AWSPollyTextToSpeechConfigz@Unable to map the custom llm provider={} to a known provider={}.r   )?r   rS   r   r   r  r   r1  r%  r   rF   Z"get_provider_text_to_speech_configr   Zmap_openai_paramsr   r:  r^  rs  r9  r   r  r(  rZ   r   rj  r   r+  r  ru  audio_speechrd  Z0litellm.llms.azure.text_to_speech.transformationr.  Zdispatch_text_to_speechre  r  ra  rc  Z5litellm.llms.elevenlabs.text_to_speech.transformationr/  stripZELEVENLABS_QUERY_PARAMS_KEYrF  ZELEVENLABS_VOICE_ID_KEYZtext_to_speech_handlerZ4litellm.llms.vertex_ai.text_to_speech.transformationr1  r<   Z4endpoints.speech.speech_to_completion_bridge.handlerr2  r)  rW  r1  r4  r7  Z3litellm.llms.runwayml.text_to_speech.transformationr3  Z2litellm.llms.minimax.text_to_speech.transformationr4  Z4litellm.llms.aws_polly.text_to_speech.transformationr7  r@  r  Zprovider_list)2r   rq  r*  r   r(  r  r   r+  r   r  r   r  r,  r-  r   r  r  r(  r   r  r  r  r  r  r  r  r  r  r0  ru  r   r.  Zazure_configr  r  r/  Zelevenlabs_configr5  Zquery_paramsr1  Zgeneric_optional_paramsr2  Zvertex_configr3  Zrunwayml_configr4  Zminimax_configZ	voice_strr7  Zaws_polly_configr   r   r   r)    s  

		






















"


r)  r   model_paramsr  )r   r   r  r8  Zaudio_transcriptionZimage_generationZvideo_generationbatchZrerankZrealtimer  Zocrc              
      s*  ddl m} ddlm} | }|dg ddtt tj tt dd}|| d	< |j	| d
} z| 
dd}|du r@td|tjv rQ|du rQtj| 
d}| 
dd}	| 
dd}
| 
dd}t||	|
|d\}}}}|tjv r|du rtj| 
d}ddi| d< |pd}d|v r|j||| |dI dH W S |j||| ||d}||v r||  I dH }t|di 
di pi }t|W S td| d ty } z>t }t|tr|dd }|du rddt| d | iW  Y d}~S t|d! | }|j
d"}||d#W  Y d}~S d}~ww )$a   
    Support health checks for different providers. Return remaining rate limit, etc.

    Returns:
        {
            "x-ratelimit-remaining-requests": int,
            "x-ratelimit-remaining-tokens": int,
            "x-ms-region": str,
        }
    r   )get_litellm_logging_class)HealthCheckHelpersr  Fr   T)r   r   r   Z	call_typer  
start_timeZfunction_idZlog_raw_request_responser  )r:  r   Nzmodel not setr  r  r(  r   r  zno-cachecacher   *)r   r  r:  r  )r   r  r:  rO  rq  r  r  zMode zI not supported. See modes here: https://docs.litellm.ai/docs/proxy/healthi  errorzerror:zz. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models  
stacktrace: z
stack trace: raw_request_typed_dict)rA  rB  )Z)litellm.litellm_core_utils.cached_importsr<  Z/litellm.litellm_core_utils.health_check_helpersr=  r   r   uuid4datetimenowZ;_update_model_params_with_health_check_tracking_informationr   r@  r   Z
model_costrS   Zahealth_check_wildcard_modelsZget_mode_handlersra  r*   	traceback
format_excr9  r$  )r:  r  rO  rq  r<  r=  r    r  r   Zcustom_llm_provider_from_paramsZapi_base_from_paramsZapi_key_from_paramsr  rJ  Zmode_handlersr  Z_response_headersrO  Zstack_traceZerror_to_returnrB  r   r   r   ahealth_check  s   





rH  c                 C   s:   zt |  tjrt|  W d S W d S  ty   Y d S w r   )rh   r=  r   Zset_verboseprintr@  )Zprint_statementr   r   r   rP    s   
rP  c                  K   s.   t jd urtt j}tdi | |S td)Nz`No config path set, please set a config path using `litellm.config_path = 'path/to/config.json'`r   )r   Zconfig_pathr^   r   rU  )r   Zconfig_argsr   r   r   config_completion  s   

rJ  chunksc              
   C   s  | d d }| d d }| d d }| d d }| d  dd }| d d d d	 }| d d d d
 }|||||d d||dgd d d dd}	g }
| D ]%}|d }|D ]}|d urpt|drp| dd urp| d}|
| qTqLd|
}||	d d d< t|dkrn	 zt||d|	d d< W n ty   td d|	d d< Y nw t||dd|	d d< |	d d |	d d  |	d d< tdi |	S )Nr   r>  r  r  r   system_fingerprintr  finish_reasonr  )rV  rx  r  rN  r|  )r>  r  r  r   rL  r  r{  rV  r  r   r{  r}  z1token_counter failed, assuming prompt tokens is 0T)r   rV  Zcount_response_tokensr~  r  r   )	r   r[  r  rl  r<  ra   r@  rP  rH   )rK  r   r>  r  r  r   rL  rN  r  r   Zcontent_listchunkr  choicer  Zcombined_contentr   r   r   $stream_chunk_builder_text_completion   st   	




rQ  ru  r    c           %   
   C   sT  z| d u rt jddddd| sW d S t| |}|j} t| dkr%W d S t| d d d t jjr9t| |dW S | d d }|	| }g }d	}	| D ]}
t|
d dkrUqJ|
d d }t|t
rf|d
i nt|d
i }t|t
rt|}nt|drttttf | }ni }|dd us|dd us|dd us|dd us|dd us|dd us|dd us|dd urd}	 n|d}t|tr|r|| qJ|	rC|rd||d d d d< t|}|j| |||dd}t|d| t| D ]-}
t|
t
r|
d}nt|
dd }t|t
r,d|v r,|jdi |d   nqt jr@|d ur@t|d|j|d |W S dd | D }t|dkrf||}tt|j d }d |j!_"||j!_#dd | D }t|dkrtt|j d }d |j!_"|$||j!_%dd | D }t|dkr|&||d d d d< d d | D }t|dkr|'||d d d d< d!d | D }t|dkr|(||d d d d< d"d | D }t|dkr|d d d d
 d }||d d d d< d#d | D }t|dkrtt|j d }|)||j!_*d$d | D }t|dkrFg }|D ]}
|+|
d d d
 d  q+||d d d d< d%d | D }t|dkri }|D ]@}
|
d d d
 d } t| t
r| , D ](\}!}"|!|vr}|"||!< qnt|"t-rt||! t-r|"||!< qn|"||!< qnqX|rtt|j d }||j!_.t|}|/|}#|j| ||||#d}t|d| t| D ].}
t|
t
r|
d}nt|
dd }t|t
rd|v r|jdi |d   nqt jr|d urt|d|j|d |W S  t0y) }$ zt12d&3t|$ t jdddddd }$~$ww )'NrU  z=Error building chunks for logging/streaming usage calculationr  )rT  r[  rW  r   r   r  )rK  r   r   Tdeltar  r  r   reasoning_contentthinking_blocksannotationsr   imagesprovider_specific_fieldsFrv  r[  )rK  r   completion_outputr   reasoning_tokensr{  r  Zcost)r  c                 S   L   g | ]"}t |d  dkr$d|d  d d v r$|d  d d d dur|qS )r  r   r  rR  Nr<  ry  rO  r   r   r   rz        z(stream_chunk_builder.<locals>.<listcomp>c                 S   rZ  )r  r   r   rR  Nr[  r\  r   r   r   rz    r]  c                 S   rZ  )r  r   rv  rR  Nr[  r\  r   r   r   rz    r]  c                 S   rZ  )r  r   rT  rR  Nr[  r\  r   r   r   rz    r]  c                 S   rZ  )r  r   rS  rR  Nr[  r\  r   r   r   rz    r]  c                 S   rZ  )r  r   rU  rR  Nr[  r\  r   r   r   rz    r]  c                 S   rZ  )r  r   r   rR  Nr[  r\  r   r   r   rz  '  r]  c                 S   rZ  )r  r   rV  rR  Nr[  r\  r   r   r   rz  5  r]  c                 S   rZ  )r  r   rW  rR  Nr[  r\  r   r   r   rz  F  r]  zAlitellm.main.py::stream_chunk_builder() - Exception occurred - {})4r   r_  rx   rK  r<  r9  r  rG   rQ  Zbuild_base_responserF  r   ra  r[  r   r   r   r	   r  r  rl  r.   Zcalculate_usager  reversedr  r  rW  Zinclude_cost_in_streaming_usageZ_response_cost_calculatorZget_combined_tool_contentrA   r  r[  rv  r  Z"get_combined_function_call_contentr   Zget_combined_contentZget_combined_thinking_contentZget_combined_reasoning_contentZget_combined_audio_contentr   extendr   rk  rW  Zcount_reasoning_tokensr@  rh   	exceptionr  )%rK  r   r>  end_timeru  	processorr   r   Zsimple_content_partsZis_simple_text_streamrO  rP  Z	delta_objrR  rv  rX  r{  ZhiddenZtool_call_chunksZtool_calls_listr  Zfunction_call_chunksZcontent_chunksrT  Zreasoning_chunksZannotation_chunksrU  Zaudio_chunksZimage_chunksZ
all_imagesZprovider_specific_chunksZcombined_provider_fieldsfieldsr  r  rY  rO  r   r   r   stream_chunk_builderj  s  










rd  _encoding_cachec                  C   s    t du rddl} | jt ja t S )z*Get encoding, loading it lazily if needed.Nr   )re  sysmodulesr   r  )rf  r   r   r   r_    s   r_  r6  c                 C   sN   | dkrddl m} | }ddl}||jt jd< |a|S tdtd| )z#Lazy import handler for main moduler  r   )_get_default_encodingNzmodule z has no attribute )Zlitellm._lazy_importsrh  rf  rg  r   __dict__re  AttributeError)r6  rh  	_encodingrf  r   r   r   __getattr__  s   rl  r   )FNrp  NFNNN)NNNNNNNNNNNNNNNNNNNNN)NN)NNN)NNNNNNr   NNNNN)NNNNNNNNNNNNNNNN)r   NN)NNNN(  r7  rB  rD  r  rP  rx  randomrf  rm  rF  r  r   r  r   r   r   r  r   	functoolsr   typingr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   Zlitellm._uuidr   Zaiohttpr   Zdotenvrb  r1  r  Zpydanticr   Ztyping_extensionsr   r   r   Zlitellm.utilsr   r   r   Z*litellm.litellm_core_utils.litellm_loggingr    Zlitellm.constantsr!   r"   Zlitellm.exceptionsr#   Z"litellm.integrations.custom_loggerr$   Z#litellm.litellm_core_utils.asyncifyr%   Z,litellm.litellm_core_utils.audio_utils.utilsr&   r'   Z%litellm.litellm_core_utils.dd_tracingr(   Z8litellm.litellm_core_utils.get_provider_specific_headersr)   Z-litellm.litellm_core_utils.health_check_utilsr*   r+   r:  Z)litellm.litellm_core_utils.mock_functionsr,   r-   Z8litellm.litellm_core_utils.prompt_templates.common_utilsr.   Zlitellm.llms.base_llmr/   r0   Z)litellm.llms.base_llm.base_model_iteratorr1   Z!litellm.llms.bedrock.common_utilsr2   Z litellm.llms.cohere.common_utilsr3   Z&litellm.llms.custom_httpx.http_handlerr4   r5   Z$litellm.llms.openai_like.json_loaderr6   Z#litellm.llms.vertex_ai.common_utilsr7   r8   Zlitellm.realtime_api.mainr9   Zlitellm.secret_managers.mainr:   r;   Zlitellm.types.routerr<   Zlitellm.types.utilsr=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   Z_loggingrh   Zcaching.cachingri   rj   rk   Zlitellm_core_utils.core_helpersrl   Z!litellm_core_utils.fallback_utilsrm   rn   Z0litellm_core_utils.prompt_templates.common_utilsro   rp   rq   Z+litellm_core_utils.prompt_templates.factoryrr   rs   rt   ru   rv   rw   Z0litellm_core_utils.streaming_chunk_builder_utilsrx   Zllms.anthropic.chatry   Zllms.azure.audio_transcriptionsrz   Zllms.azure.azurer{   r|   Z llms.azure.chat.o_series_handlerr}   Zllms.azure.completion.handlerr~   Zllms.azure_ai.anthropic.handlerr   Zllms.azure_ai.embedr   Zllms.bedrock.chatr   r   Zllms.bedrock.embed.embeddingr   Zllms.bedrock.image_edit.handlerr   Z+llms.bedrock.image_generation.image_handlerr   Zllms.bytez.chat.transformationr   Z!llms.clarifai.chat.transformationr   Z!llms.codestral.completion.handlerr   Zllms.cohere.embedr   Zcohere_embedZ!llms.custom_httpx.aiohttp_handlerr   Z"llms.custom_httpx.llm_http_handlerr   Zllms.custom_llmr   r   Zllms.databricks.embed.handlerr   Zllms.deprecated_providersr   r   Zllms.gemini.common_utilsr   Zllms.groq.chat.handlerr   Zllms.heroku.chat.transformationr   Z"llms.huggingface.embedding.handlerr   Z!llms.lemonade.chat.transformationr   Zllms.nlp_cloud.chat.handlerr   r|  Zllms.oci.chat.transformationr   Zllms.ollama.completionrD  Zllms.oobabooga.chatr   Zllms.openai.completion.handlerr   Z$llms.openai.image_variations.handlerr   Zllms.openai.openair   Z"llms.openai.transcriptions.handlerr   Zllms.openai_like.chat.handlerr   Z"llms.openai_like.embedding.handlerr   Z!llms.ovhcloud.chat.transformationr   Zllms.petals.completionr  Zllms.predibase.chat.handlerr   Zllms.replicate.chat.handlerrw  Zllms.sagemaker.chat.handlerr   Z!llms.sagemaker.completion.handlerr   Zllms.sap.chat.handlerr   Zllms.vertex_air   Z8llms.vertex_ai.gemini.vertex_and_google_ai_studio_geminir   Z<llms.vertex_ai.gemini_embeddings.batch_embed_content_handlerr   Z8llms.vertex_ai.image_generation.image_generation_handlerr   Z6llms.vertex_ai.multimodal_embeddings.embedding_handlerr   Z,llms.vertex_ai.vertex_ai_partner_models.mainr   Z2llms.vertex_ai.vertex_embeddings.embedding_handlerr   Z'llms.vertex_ai.vertex_gemma_models.mainr   Z'llms.vertex_ai.vertex_model_garden.mainr   Zllms.vllm.completionr  Zllms.watsonx.chat.handlerr   Zllms.watsonx.common_utilsr   Ztypes.llms.anthropicr   Ztypes.llms.openair   r   r   r   r   r   r   r   Ztypes.utilsr   r   r   r   r   r   r   r   r   ru  rm  r'  Zopenai_image_variationsZgroq_chat_completionsro  Zsap_gen_ai_hub_embr  rz  ri  rc  rb  rd  r&  r  r  r  r  r  Zbedrock_image_generationZbedrock_image_editr  r  r  Zvertex_image_generationr  r  r  r  r  r  r  r  r  re  rp  Zsagemaker_chat_completionr  Zheroku_transformationZoci_transformationr  r  r   r@  rF  ZMOCK_RESPONSE_TYPEr   r   r   r   wrapr   r   rV  rk  r   rS  rf  ri  rk  r6  rh  rl  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r(  r)  rH  rP  rJ  rQ  rd  re  __annotations__r_  rl  r   r   r   r   <module>   s@  
	H) JR













	

"#$&'()*,-/1
3  
;""	
 (!	
 "#$&')*+,.02
4                          
,
 

" B	
	
	
        J
I* !"%&'+,-.0  )' KC	
 X(	
   a 	L  + 