o ưiʟ @sxdZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZmZddlmZddlZddlmZmZddlmZddlmZddlmZdd lmZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,erddl-m.Z/neZ/ddl0m1Z1ddl2m3Z3GdddeZ4eZ5GdddZ6 dde de eedfde e7effddZ8dS)a This contains LLMCachingHandler This exposes two methods: - async_get_cache - async_set_cache This file is a wrapper around caching.py This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc) It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup In each method it will call the appropriate method from caching.py N) TYPE_CHECKINGAnyAsyncGeneratorCallableDict GeneratorListOptionalTupleUnion) BaseModel) print_verboseverbose_logger) InMemoryCache)S3Cache)update_response_metadata)1_assemble_complete_response_from_streaming_chunks)CachedEmbedding)ResponsesAPIResponse)RerankResponse)CachingDetails CallTypes EmbeddingEmbeddingResponse ModelResponseTextCompletionResponseTranscriptionResponseUsage)Logging!_get_parent_otel_span_from_kwargsCustomStreamWrapperc@s>eZdZUdZdZeeed<dZee ed<dZ e ed<dS)CachingHandlerResponsea This is the response object for the caching handler. We need to separate embedding cached responses and (completion / text_completion / transcription) cached responses For embeddings there can be a cache hit for some of the inputs in the list and a cache miss for others N cached_resultfinal_embedding_cached_responseF embedding_all_elements_cache_hit) __name__ __module__ __qualname____doc__r$r r__annotations__r%rr&boolr-r-V/home/app/Keep/.python/lib/python3.10/site-packages/litellm/caching/caching_handler.pyr#Hs r#c@s*eZdZdedeeefdejfddZ dBdedede dejd ed eeefd e e ed fd e e fddZ dBdedede dejd ed eeefd e e ed fd e fddZd eeefd eefddZdee eefd e efddZde edee ed eeefde dejded e e eeffddZdeded efddZde d edejd!ejd ef d"d#Zde dedejd!ejd$ef d%d&Zd ed eeefd e ed fd e efd'd(Z dBded ed eeefde ded e ed fd)e ed e eeeeee e!ffd*d+Z"ded ede ded e!f d,d-Z# dBd.eded eeefd e e ed ffd/d0Z$ dBd.ed eeefd e e ed ffd1d2Z%ded eeefd efd3d4Z&ded efd5d6Z'd7efd8d9Z(d7efd:d;Z) <  dCde ded eeefded=ed>ed)e ed?e e*fd@dAZ+dS)DLLMCachingHandleroriginal_functionrequest_kwargs start_timecCsfddlm}m}g|_g|_||_||_||_tj dur.t tj j |r.|tj j t d|_ dSd|_ dS)Nr) DualCache RedisCache) redis_cacheZin_memory_cache) litellm.cachingr3r4async_streaming_chunkssync_streaming_chunksr1r0r2litellmcache isinstancein_memory_cache_obj dual_cache)selfr0r1r2r3r4r-r-r.__init__ZszLLMCachingHandler.__init__Nmodel logging_obj call_typekwargsargs.returnc sV|dddurtjdus|dddur)|didddur)|p'd}d}d} d} |}t} d} t|} | |d<tjdur|j|d rt d |j |||d IdH} t} | durt | t st d d}t j }tj||d d|dd|ddd\}}}}| | d}|j|||| d||d|j}|j| |||||d d|d} |dddur|j|| |||dtjjdi|}t | tst | trt| dr|| jd<t| dS|tjjkr| durt | t rtjdurt tjjts|j|| ||||d\}} t|| dSt d| t| |dSdS)aC Internal method to get from the cache. Handles different call types (embeddings, chat/completions, text_completion, transcription) and accordingly returns the cached response Args: model: str: original_function: Callable: logging_obj: LiteLLMLoggingObj: start_time: datetime.datetime: call_type: str: kwargs: Dict[str, Any]: args: Optional[Tuple[Any, ...]] = None: Returns: CachingHandlerResponse: Raises: None ZcachingNFTr:zno-cacher-parent_otel_spanr0zChecking Async Cache)rBrCrDz Cache Hit!custom_llm_providerapi_baseapi_keyr@rHrIrJ)rAr@rCr$is_asyncrHcache_duration_msr$rBrCrAr@rHrDstreamrAr$r2end_time cache_hit_hidden_params cache_keyr$)r%r$rCrAr2r@)r%r&zCACHE RESULT: )r$r%)getr9r:copytime perf_counterr _is_call_type_supported_by_cacherdebug_retrieve_from_cacher;listdatetimenowget_llm_provider'_update_litellm_logging_obj_environmentr'(_convert_cached_result_to_model_response!_async_log_cache_hit_on_callbacks get_cache_keyr r"hasattrrTr#r aembeddingvaluer(_process_async_embedding_cached_response)r>r@r0rAr2rBrCrDr%r&r$Zcache_check_start_timeZcache_check_end_timerFrSrRrH_rNrUr-r-r._async_get_cacheps                 z"LLMCachingHandler._async_get_cachec Cs\ddlm}d} tjdur|j|dr|pd}|} | t|j|t dtjj di| } | durd| vr:no|j }|j | ||||| dd|d} d } tj} tj|pYd | dd| d d| d dd \}} }}|j|| d||| dd|j| || | dtjjdi|}t| tst| |rt| dr|| jd<t| dSt| dS)Nrr!rGr-zChecking Sync CachedetailrHrOTrIrJrK/F)rAr@rCr$rMresultr2rRrSrTrUrV) litellm.utilsr"r9r:r[rXupdateconvert_args_to_kwargsr0r get_cacher'rcrWr_r`rarb-handle_sync_success_callbacks_for_async_callsrer;r rfrTr#)r>r@r0rAr2rBrCrDr"r$ new_kwargsrSrRrHdynamic_api_keyrIrUr-r-r._sync_get_caches          z!LLMCachingHandler._sync_get_cachecCs6t|dtr |dgSt|dtr|dStd)zO Handles the input of kwargs['input'] being a list or a string input input must be a string or a list)r;strr^ ValueError)r>rCr-r-r.handle_kwargs_input_list_or_strSs  z1LLMCachingHandler.handle_kwargs_input_list_or_str non_null_listcCs2|D]\}}t|tr|dr|dSqdS)a  Helper method to extract the model name from cached results. Args: non_null_list: List of (idx, cr) tuples where cr is the cached result dict Returns: Optional[str]: The model name if found, None otherwise r@N)r;dictrW)r>r~rjcrr-r-r."_extract_model_from_cached_results^s  z4LLMCachingHandler._extract_model_from_cached_resultsr%r$cCsd}g}g} ||} t|D]\} } | dur|| | q| | | fq||d<t| dkr|| } | s=|d} t| dgt| d}d|jd<d}| D]4}|\} } | dur| d }|durnt|| d d |j | <t | | t rdd l m }||| | dd 7}qQt|d|d }||_t|dkrd}d}tj}tj||dd|dd|ddd\}}}}|j||||ddd|j|||||d||fS||fS)a Returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit For embedding responses, there can be a cache hit for some of the inputs in the list and a cache miss for others This function processes the cached embedding responses and returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit Args: final_embedding_cached_response: Optional[EmbeddingResponse]: cached_result: List[Optional[Dict[str, Any]]]: kwargs: Dict[str, Any]: logging_obj: LiteLLMLoggingObj: start_time: datetime.datetime: model: str: Returns: Tuple[Optional[EmbeddingResponse], bool]: Returns the final embedding cached response and a boolean indicating if all elements in the list have a cache hit FNryrr@)r@dataTrS embedding)rindexobject) token_counter)textZcount_response_tokens prompt_tokenscompletion_tokens total_tokensrHrIrJrK)rAr@rCr$rM is_embeddingrQ)r} enumerateappendlenrrWrrTrrr;r{rqrrusager_r`r9rarbrd)r>r%r$rCrAr2r@r&Zremaining_listr~Zkwargs_input_as_listidxrZ model_namervalZembedding_datarrrSrRrHrwrIr-r-r.rios              z:LLMCachingHandler._process_async_embedding_cached_responseusage1usage2cCs&t|j|j|j|j|j|jdS)Nr)rrrr)r>rrr-r-r. combine_usages    zLLMCachingHandler.combine_usage_caching_handler_responseembedding_responserRcCs|jdur|Sd}g}|jjD]}|dur'|jdur'||j||d7}q||q||j_d|jjd<||d|j_|jjdurX|jdurX|j|jj|jd|j_|jS)a Combines the cached embedding response with the API EmbeddingResponse For caching there can be a cache hit for some of the inputs in the list and a cache miss for others This function combines the cached embedding response with the API EmbeddingResponse Args: caching_handler_response: CachingHandlerResponse: embedding_response: EmbeddingResponse: Returns: EmbeddingResponse: NrTrSrL)rr)r%rrrT total_secondsZ _response_msrr)r>rrr2rRrZfinal_data_listitemr-r-r.2_combine_cached_embedding_response_with_api_results0       zDLLMCachingHandler._combine_cached_embedding_response_with_api_resultrScCs<ddlm}|j|j||||dd|j||||ddS)a Helper function to log the success of a cached result on callbacks Args: logging_obj (LiteLLMLoggingObj): The logging object. cached_result: The cached result. start_time (datetime): The start time of the operation. end_time (datetime): The end time of the operation. cache_hit (bool): Whether it was a cache hit. r)GLOBAL_LOGGING_WORKERro)Zasync_coroutineN)Z)litellm.litellm_core_utils.logging_workerrZensure_initialized_and_enqueueZasync_success_handlerru)r>rAr$r2rRrSrr-r-r.rds  z3LLMCachingHandler._async_log_cache_hit_on_callbacksc sFtjdurdS|}|t|j|d}|tjjkr}t |dt r,|dg|d<n t |dt s7t dg}t |dD]\}}tjjd ii|d|i} |tjj| |jdq?tj|IdH}|dur{t |t r{tdd|Dr{d}|Stjdurtjjd d|ji|IdH}|Stjjd d|ji|}|S) a Internal method to - get cache key - check what type of cache is used - Redis, RedisSemantic, Qdrant, S3 - async get cache value - return the cached value Args: call_type: str: kwargs: Dict[str, Any]: args: Optional[Tuple[Any, ...]] = None: Returns: Optional[Any]: Raises: None Nryrz)rUdynamic_cache_objectcss|]}|duVqdSNr-).0rpr-r-r. hsz9LLMCachingHandler._retrieve_from_cache..Trr-)r9r:rXrrrsr0rrgrhr;r{r^r|rrerZasync_get_cacher=asynciogatherallZ_supports_asyncrt) r>rBrCrDrvr$tasksripreset_cache_keyr-r-r.r]5sZ       z&LLMCachingHandler._retrieve_from_cacherHc Csddlm}|tjjks|tjjkr0t|tr0|dddur)|j ||||d}n||t d}|tj jks<|tj jkr[t|tr[|dddurS|j ||||d}npt di|}nh|tjjksg|tjjkrut|tru||tdd }nN|tjjks|tjjkrt|tr||d d d }n5|tjjks|tjjkrt|trd |dd } ||td| d}n|dks|dkrt|trtdi|}t|dr|jd urt|jtrd|jd<t|||||jtjd|S)aH Internal method to process the cached result Checks the call type and converts the cached result to the appropriate model response object example if call type is text_completion -> returns TextCompletionResponse object Args: cached_result: Any: call_type: str: kwargs: Dict[str, Any]: logging_obj: LiteLLMLoggingObj: model: str: custom_llm_provider: Optional[str] = None: args: Optional[Tuple[Any, ...]] = None: Returns: Optional[Any]: r) convert_to_model_response_objectrPFT)r$rBrAr@)response_objectmodel_response_objectr)rr response_typeNrerankz whisper-1)r@rHrSZaudio_transcription)rrr hidden_paramsZ aresponses responsesrTrS)rprAr@rCr2rRr-)rqrr acompletionrh completionr;rrW_convert_cached_stream_responseratext_completionZtext_completionrrgrrZarerankrZatranscriptionZ transcriptionrrrfrTrr2r_r`) r>r$rBrCrAr@rDrHrrr-r-r.rcvs %           z:LLMCachingHandler._convert_cached_result_to_model_responsec CsRddlm}m}m}|tjjks|tjjkr||d}n||d}|||d|dS)Nr)r"convert_to_streaming_response#convert_to_streaming_response_async)rcached_response)Zcompletion_streamr@rHrA)rqr"rrrrrhr) r>r$rBrAr@r"rrZ_stream_cached_resultr-r-r.rs  z1LLMCachingHandler._convert_cached_stream_responserpcsddlm}tjdurdS|}|t||||}||d<|j||drt|tj sDt|tj sDt|t sDt|t sDt|t r{t|t rgtjdurgttjjtsgttjj|fd|ji|dSttjj|fd|ji|dSttjj|fi|dSdS)ah Internal method to check the type of the result & cache used and adds the result to the cache accordingly Args: result: Any: original_function: Callable: kwargs: Dict[str, Any]: args: Optional[Tuple[Any, ...]] = None: Returns: None Raises: None rrNrFr0rCr)'litellm.litellm_core_utils.core_helpersr r9r:rXrrrs_should_store_result_in_cacher;rrrrrrr create_taskZasync_add_cache_pipeliner=Zasync_add_cacheZmodel_dump_json)r>rpr0rCrDr rvrFr-r-r.async_set_cachesj     z!LLMCachingHandler.async_set_cachecCsP|}|t|j|tjdurdS|j|j|dr&tjj|fi|dS)zE Sync internal method to add the result to the cache Nr)rXrrrsr0r9r:rZ add_cache)r>rprCrDrvr-r-r.sync_set_cache]s  z LLMCachingHandler.sync_set_cachecCs@tjduotjjduot|jtjjvo|diddduS)z Helper function to determine if the result should be stored in the cache. Returns: bool: True if the result should be stored in the cache, False otherwise. Nr:zno-storeFT)r9r:supported_call_typesr{r'rW)r>r0rCr-r-r.rxs z/LLMCachingHandler._should_store_result_in_cachecCs0tjdurtjjdurt|jtjjvrdSdS)aO Helper function to determine if the call type is supported by the cache. call types are acompletion, aembedding, atext_completion, atranscription, arerank Defined on `litellm.types.utils.CallTypes` Returns: bool: True if the call type is supported by the cache, False otherwise. NTF)r9r:rr{r')r>r0r-r-r.r[s  z2LLMCachingHandler._is_call_type_supported_by_cacheprocessed_chunkcsLt||jtj|j|jdd}|dur$|j||j|jdIdHdSdS)z Internal method to add the streaming response to the cache - If 'streaming_chunk' has a 'finish_reason' then assemble a litellm.ModelResponse object - Else append the chunk to self.async_streaming_chunks Trpr2rRr1Zstreaming_chunksrMN)rpr0rC)rr2r_r`r1r7rr0r>rZcomplete_streaming_responser-r-r. _add_streaming_response_to_caches"  z2LLMCachingHandler._add_streaming_response_to_cachecCs@t||jtj|j|jdd}|dur|j||jddSdS)zQ Sync internal method to add the streaming response to the cache FrN)rprC)rr2r_r`r1r8rrr-r-r.%_sync_add_streaming_response_to_caches  z7LLMCachingHandler._sync_add_streaming_response_to_cacheFrMrrNc Cs|dd||dd|di|di|dd|di|d } tjdur4tjjdi|| d <nd| d <|j||d di| |sJ|d dn|d d|ddt|d|dd|d td|d|_dS)a Helper function to update the LiteLLMLoggingObj environment variables. Args: logging_obj (LiteLLMLoggingObj): The logging object to update. model (str): The model being used. kwargs (Dict[str, Any]): The keyword arguments from the original function call. cached_result (Any): The cached result to log. is_async (bool): Whether the call is asynchronous or not. is_embedding (bool): Whether the call is for embeddings or not. custom_llm_provider (Optional[str]): The custom llm provider being used. Returns: None logger_fnNrIrmmetadata model_infoproxy_server_requeststream_response)rrrIrrrrrHrusermessagesryrJrPF) r@rZoptional_paramslitellm_paramsryrJoriginal_responseZadditional_argsrPrHT)rSrNr-)rWr9r:Z!_get_preset_cache_key_from_kwargsZupdate_environment_variablesr{rZcaching_details) r>rAr@rCr$rMrrHrNrr-r-r.rbs>            z9LLMCachingHandler._update_litellm_logging_obj_environmentr)FNN),r'r(r)rrr{rr_r?LiteLLMLoggingObjr r r#rkrxrr}intrrrr,rirrrrdr]r rrrrr"rcrrrrr[rrfloatrbr-r-r-r.r/Ys|      S     m 4    I     %  K       r/r0rD.rEcCsRt|}t|j}i}|r't|D]\}}|t|kr&||}|||<q|Sr)inspect signaturer^ parameterskeysrr)r0rDrZ param_namesZargs_to_kwargsrarg param_namer-r-r.rss  rsr)9r*rr_rrYtypingrrrrrrrr r r Zpydanticr r9Zlitellm._loggingr rr6rZlitellm.caching.cachingrZ?litellm.litellm_core_utils.llm_response_utils.response_metadatarZ(litellm.litellm_core_utils.logging_utilsrZlitellm.types.cachingrZlitellm.types.llms.openairZlitellm.types.rerankrZlitellm.types.utilsrrrrrrrrZ*litellm.litellm_core_utils.litellm_loggingrrrr Z,litellm.litellm_core_utils.streaming_handlerr"r#r<r/r{rsr-r-r-r.sP0        (   C