o ưi@sddlZddlmZddlmZddlmZmZmZm Z m Z m Z m Z m Z mZddlZddlZddlmZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZm Z dd l!m"Z"dd l#m$Z$m%Z%GdddeZ&Gddde'eZ(dde(DZ) dmde'de e*de(fddZ+de'de dde,fddZ-de'de dde,fddZ.de'de,fdd Z/dd!lm Z m Z e d"Z0de'de'fd#d$Z1d%e e'de'fd&d'Z2de'd(e e'd%e e'd)e d*de e'e'ff d+d,Z3d-e0de'd.e e,d(e e'd%e e'd)e d*de e'e'ffd/d0Z4d-e0de'd.e e,d1e e'de e'e'ff d2d3Z5d4eede,fd5d6Z6dnd7d8Z7dnd9d:Z8dodd?Z9d dmdBee'efdKe e'dee'effdLdMZ?dndNdOZ@dPdQZAdRe'fdSdTZBdUe'de=fdVdWZCdndXdYZDdZe'de e'fd[d\ZEdZe'de e'fd]d^ZFdZe'de e'fd_d`ZGdae'd(e'd%e'de'fdbdcZHdde'dae'd%e e'd(e e'dejIf dedfZJde'de,fdgdhZKGdidjdjeZLGdkdldleZMdS)pN)deepcopy)Enum) AnyDictListLiteralOptionalSetTupleUnionget_type_hints)verbose_logger)DEFAULT_MAX_RECURSE_DEPTH) unpack_defs)BaseLLMModelInfoBaseTokenCounter)BaseLLMException)AllMessageValues)PartTypeSchema)TokenCountResponse)supports_response_schemasupports_system_messagesc s<eZdZ ddededeeeej fffdd Z Z S) VertexAIErrorN status_codemessageheaderscstj|||ddS)N)rrr)super__init__)selfrrr __class__Z/home/app/Keep/.python/lib/python3.10/site-packages/litellm/llms/vertex_ai/common_utils.pyrszVertexAIError.__init__N) __name__ __module__ __qualname__intstrrr rhttpxZHeadersr __classcell__r"r"r r#rsrc@s0eZdZdZdZdZdZdZdZdZ dZ d Z d S) VertexAIModelRoutez Enum for Vertex AI model routingZpartner_modelsgeminiZgemmabgeZ model_gardenZ non_geminiopenaiZ agent_engineN) r%r&r'__doc__PARTNER_MODELSGEMINIGEMMABGE MODEL_GARDEN NON_GEMINIZOPENAI_COMPATIBLE AGENT_ENGINEr"r"r"r#r,sr,cCsg|]}|jdqS)/)value).0router"r"r# *sr<modellitellm_paramsreturncCsddlm}|r|ddurd|dvrtjSd|vrtjS|r-|r-|dr-tjS|j|dr6tjSd |vs@d | vrCtj Sd |vrJtj Sd |vrQtj Sd|vrXtjStj S) a Determine which handler to use for a Vertex AI model based on the model name. Args: model: The model name (e.g., "llama3-405b", "gemini-pro", "gemma/gemma-3-12b-it", "openai/gpt-oss-120b") litellm_params: Optional litellm parameters dict that may contain base_model for routing Returns: VertexAIModelRoute: The route enum indicating which handler should be used Examples: >>> get_vertex_ai_model_route("llama3-405b") VertexAIModelRoute.PARTNER_MODELS >>> get_vertex_ai_model_route("gemini-pro") VertexAIModelRoute.GEMINI >>> get_vertex_ai_model_route("gemma/gemma-3-12b-it") VertexAIModelRoute.GEMMA >>> get_vertex_ai_model_route("openai/gpt-oss-120b") VertexAIModelRoute.MODEL_GARDEN >>> get_vertex_ai_model_route("1234567890", {"api_base": "http://10.96.32.8"}) VertexAIModelRoute.GEMINI # Numeric endpoints with api_base use HTTP path rVertexAIPartnerModelsZ base_modelNr-z agent_engine/api_baser=zbge/r.zgemma/r/)4litellm.llms.vertex_ai.vertex_ai_partner_models.mainrAgetr,r2r7isdigitis_vertex_partner_modelr1lowerr4r3r5r6)r=r>rAr"r"r#get_vertex_ai_model_route,s&   rIcustom_llm_provider) vertex_aivertex_ai_betar-c Csxz|}|dkr d}t||d}tj|rd}W|SW|Sty;}ztdt|d}WYd}~|Sd}~ww)NrLrKr=rJTzUnable to identify if system message supported. Defaulting to 'False'. Received error message - {} Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.jsonF) rlitellmVertexGeminiConfigZ_is_model_gemini_spec_model Exceptionr warningformatr))r=rJ_custom_llm_providerZsupports_system_messageer"r"r#get_supports_system_messagess,   rUcCs |}|dkrd}t||d}|S)NrLrKrM)r)r=rJrSZ_supports_response_schemar"r"r#get_supports_response_schemasrVcCs |}td}t||S)a Check if the model supports responseJsonSchema (JSON Schema format). responseJsonSchema is supported by Gemini 2.0+ models and uses standard JSON Schema format with lowercase types (string, object, etc.) instead of the OpenAPI-style responseSchema with uppercase types (STRING, OBJECT, etc.). Benefits of responseJsonSchema: - Supports additionalProperties for stricter schema validation - Uses standard JSON Schema format (no type conversion needed) - Better compatibility with Pydantic's model_json_schema() Args: model: The model name (e.g., "gemini-2.0-flash", "gemini-2.5-pro") Returns: True if the model supports responseJsonSchema, False otherwise zgemini-([2-9]|[1-9]\d+)\.)rHrecompileboolsearch)r=Z model_lowerZgemini_2_plus_patternr"r"r#supports_response_json_schemas r[)rr)chat embeddingbatch_embeddingimage_generation count_tokenscCs*tD]}||r||ddSq|S)a Strip routing prefixes from model name for PSC/endpoint URL construction. Patterns like "bge/", "gemma/", "openai/" are used for internal routing but should not appear in the actual endpoint URL. Routing prefixes are derived from VertexAIModelRoute enum values. Args: model: The model name with potential prefix (e.g., "bge/123456", "gemma/gemma-3-12b-it") Returns: str: The model name without routing prefix (e.g., "123456", "gemma-3-12b-it") Examples: >>> get_vertex_base_model_name("bge/378943383978115072") "378943383978115072" >>> get_vertex_base_model_name("gemma/gemma-3-12b-it") "gemma-3-12b-it" >>> get_vertex_base_model_name("openai/gpt-oss-120b") "gpt-oss-120b" >>> get_vertex_base_model_name("1234567890") "1234567890" )VERTEX_AI_MODEL_ROUTES startswithreplace)r=r;r"r"r#get_vertex_base_model_names  rfvertex_locationcCs|dkrdSd|dS)z3 Get the base URL for Vertex AI API calls. globalz!https://aiplatform.googleapis.comzhttps://z-aiplatform.googleapis.comr")rgr"r"r#get_vertex_base_urls rivertex_projectvertex_api_version)v1v1beta1c Cstd}t|d}t|}|r&|d|d|d|d|d| }||fS|d|d|d |d| }||fS) z Get URL for embedding models. Handles special patterns: - bge/endpoint_id -> strips to endpoint_id for endpoints/ routing - numeric model -> routes to endpoints/ - regular model -> routes to publishers/google/models/ predictrCr8 /projects/ /locations/ /endpoints/: /v1/projects//publishers/google/models/)rfrirF)r=rjrgrkendpointbase_urlurlr"r"r#_get_embedding_urls & rxmodestreamc Cstd}d}tjj|d}|dkrOd}t|}|durd}|r3|d|d|d|d |d | }n|d|d|d|d |d | }|durN|d 7}n\|d kr[t||||dS|dkrd}t|}|r}|d|d|d|d |d | }n.|d|d|d |d | }n|dkrd}t|}|d|d|d|d |d | }|r|std|||fS)NrCr\generateContentTstreamGenerateContentr8rorprqrrrtz?alt=sser])r=rjrgrkr_rnrsr` countTokensz,Unable to get vertex url/endpoint for mode: )rNrOZget_model_for_vertex_ai_urlrirFrx ValueError) ryr=rzrjrgrkrwrurvr"r"r#_get_vertex_urlsD(&("&rgemini_api_keyc Csddlm}d|}||rdnd}|dkr8d}|dur,d }d ||||}||fSd ||||}||fS|d krId }d|||}||fS|dkrZd}d|||}||fS|dkrkd}d|||}||fS|dkrstdtd|)Nr)rOz models/{}Zv1alphaZv1betar\r{Tr|zAhttps://generativelanguage.googleapis.com/{}/{}:{}?key={}&alt=ssez9https://generativelanguage.googleapis.com/{}/{}:{}?key={}r]Z embedContentz=https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}r^ZbatchEmbedContentsr`r}r_zLiteLLM's `gemini/` route does not support image generation yet. Let us know if you need this feature by opening an issue at https://github.com/BerriAI/litellm/issueszUnsupported mode: )Z@litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_geminirOrRZ_is_gemini_3_or_newerr~) ryr=rzrrOZ_gemini_model_name api_versionrurwr"r"r#_get_gemini_urlJsN   rpartscCs,d}|D]}d|vr|ddurd}q|S)a# check that user_content has 'text' parameter. - Known Vertex Error: Unable to submit request because it must have a text parameter. - 'text' param needs to be present (empty strings are valid) - Relevant Issue: https://github.com/BerriAI/litellm/issues/5515 FtextNT)rE)rZhas_text_parampartr"r"r#_check_text_in_content}s rcCs|tkr tdtdd|vr"t|dtr"dd|dD|d<|dd}|dur=|D] \}}t||dd q0|d d}|durQt||dd dSdS) zkFix empty strings in enum values by replacing them with None. Gemini doesn't accept empty strings in enums. Max depth of " exceeded while processing schema.enumcSsg|] }|dkr dn|qS)raNr")r:r9r"r"r#r<sz+_fix_enum_empty_strings.. propertiesNrbdepthitems)rr~ isinstancelistrEr_fix_enum_empty_strings)schemarr_r9rr"r"r#rs   rc Csn|tkr tdtdt|tsdSd|vrct|dtrc|d}d}t|tr3|dkr3d}n(|d }t|tr[|D]}t|trZ|d}t|trZ|dkrZd}nq?|sc|dd|d d}|dur~| D] \}} t | |d d qq|d d} | durt | |d d |d d}|durt|tr|D]}t|trt ||d d qdSdSdS)a+Remove `enum` fields when the schema type is not string. Gemini / Vertex APIs only allow enums for string-typed fields. When an enum is present on a non-string typed property (or when `anyOf` types do not include a string type), remove the enum to avoid provider validation errors. rrNrtypeFstringTanyOfrrbrr) rr~rdictrrEr)rHpopr_fix_enum_types) rrZ schema_typeZ keep_enumanyofitemZ item_typerrr9rr"r"r#rsJ            rF parametersadd_property_orderingcCspttt}|di}t||t|t|t|t |t |t |t ||}|r6t ||S)a This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419 Updates the input parameters, removing extraneous fields, adjusting types, unwinding $defs, and adding propertyOrdering if specified, returning the updated parameters. Parameters: parameters: dict - the json schema to build from add_property_ordering: bool - whether to add propertyOrdering to the schema. This is only applicable to schemas for structured outputs. See set_schema_property_ordering for more details. Returns: parameters: dict - the input parameters, modified in place z$defs)setr rkeysrrconvert_anyof_null_to_nullable_convert_schema_typesrr process_itemsadd_object_typefilter_schema_fieldsset_schema_property_ordering)rrZvalid_schema_fieldsZdefsr"r"r#_build_vertex_schemas   rcCs|S)a3 Build a JSON Schema for use with Gemini's responseJsonSchema parameter. Unlike _build_vertex_schema (used for responseSchema), this function: - Does NOT convert types to uppercase (keeps standard JSON Schema format) - Does NOT add propertyOrdering - Does NOT filter fields (allows additionalProperties) - Preserves $defs/$ref (Gemini 2.0+ supports JSON Schema references natively) Parameters: parameters: dict - the JSON schema to process Returns: dict - the processed schema in standard JSON Schema format r")rr"r"r#_build_json_schemasr schema_dictcCs|dd}|dd}t|trA|drA|d}|s|r=t|tr=tdd|Dr=|D]}|r6||d<|r<||d<q.d|iS|S)aM When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164 Filter out other fields in the same dict. E.g. {"anyOf": [{"type": "string"}, {"type": "null"}], "default": "test"} -> {"anyOf": [{"type": "string"}, {"type": "null"}]} Case 2: If additional metadata is present, try to keep it E.g. {"anyOf": [{"type": "string"}, {"type": "null"}], "default": "test", "title": "test"} -> {"anyOf": [{"type": "string", "title": "test"}, {"type": "null", "title": "test"}]} titleN descriptionrcss|]}t|tVqdSr$)rrr:rr"r"r# 3sz'_filter_anyof_fields..)rErrrall)rrrany_ofrr"r"r#_filter_anyof_fields!s( rcCs|tkr tdtdt|trMd|vr!|dikr!ddi|d<|D])\}}t|tr6t||dq%t|trL|D]}t|trKt||dq=q%dSdS)NrQ exceeded while processing schema. Please check the schema for excessive nesting.rrobjectrb)rr~rrrrr)rrkeyr9rr"r"r#r>s"      rrrcCs|tkr tdtdd|vr:t|dtr:d|vr(dd|dD|d<|dD] \}}t||dq.d|vrGt|d|d|S) ay vertex ai and generativeai apis order output of fields alphabetically, unless you specify the order. python dicts retain order, so we just use that. Note that this field only applies to structured outputs, and not tools. Function tools are not afflicted by the same alphabetical ordering issue, (the order of keys returned seems to be arbitrary, up to the model) https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.cachedContents#Schema.FIELDS.property_ordering Args: schema: The schema dictionary to process depth: Current recursion depth to prevent infinite loops rrrZpropertyOrderingcSsg|]\}}|qSr"r"r:kvr"r"r#r<dsz0set_schema_property_ordering..rbr)rr~rrrr)rrrrr"r"r#rOs  r valid_fieldscs durtt|}|vr|S|t|ts|Si}t|}|D][\}}|vr0q'|dkrHt|trHfdd|D||<q'|dkrV|dvrU|||<q'q'|dkrht|trht|||<q'|dkr~t|tr~fd d |D||<q'|||<q'|S) zK Recursively filter a schema dictionary to keep only valid fields. Nrcsi|] \}}|t|qSr"rr processedrr"r# s z(filter_schema_fields..rR>z date-timerrrcsg|]}t|qSr"rrrr"r#r<s z(filter_schema_fields..) ridaddrrrrrr)rrrZ schema_idresultrr9r"rr#rls:       rc Cs4|tkr tdtd |dd}|durid}|D]$}t|tr1|ddkr1||d}qd|vr?t|dkr?d |d<qt|dkrJtd |ri|D]}|dd krdd |vrd|d sd|d d|d <qN|dd}|dur|D] \}}t ||ddqw|d d}|durt ||dddSdS)NrrrFrnullTrrzaInvalid input: AnyOf schema with only null type is not supported. Please provide a non-null type.arrayrZnullablerrbr) rr~rErrremovelenrrr) rrrZ contains_nullZatypernamer9rr"r"r#rsF        rcCsd|vrd|vrd|vrd|vrd|d<|dd}|durRd|vr.|ddur.|dd|sA|dd|ddd|d<nd|d<|D]\}}t|qI|dd}|dur`t|d D]}||d}|durt|tr|D] }t|trt|quqbdS) NrroneOfallOfrrrequiredr)rrr)rErrrrrr)rrrr9rrvaluesr"r"r#rs0          r field_namecCsb||d|dd}|dur|D] \}}t||q|dd}|dur/t||dSdS)Nrr)rrEr strip_field)rrrrr9rr"r"r#rs    rvertex_datetimecCs$ddlm}||d}t|S)z Converts a Vertex AI datetime string to an OpenAI datetime integer vertex_datetime: str = "2024-12-04T21:53:12.120184Z" returns: int = 1722729192 r)datetimez%Y-%m-%dT%H:%M:%S.%fZ)rstrptimer( timestamp)rrdtr"r"r#+_convert_vertex_datetime_to_openai_datetimes   rc Cs|tkr tdtdt|tsdSd|vr|d}t|trt|dkrhd}g}|D]:}t|ts6q.|dkrB|ddiq.|dvrad|i}|D]}||vrZt||||<qL||q.|d|iq.t d d |D}|r|D]}| |dqv||d <| dnt|trt|dkr|d |d<n t|tr||d<d D]C} | |vr|| } | dkrt| tr| D] } t | |dqq| dkrt | |dq| d krt| tr| D] } t | |dqqdS)z Convert type arrays and lowercase types for Vertex AI compatibility. Transforms OpenAI-style schemas to Vertex AI format by converting type arrays like ["string", "number"] to anyOf format and converting all types to uppercase. rrNrrb>Z maxPropertiesZadditionalPropertiesrZminItemsrZ minPropertiesZmaxItemsrrrrcss"|] }t|tr|dvVqdS)rN)rr))r:tr"r"r#r3s z(_convert_schema_types..rr)rrrrr) rr~rrrrr)appendranyrrr) rrZtype_valZtype_specific_fieldsrrZ item_schemafieldZhas_object_or_arrayrr9Z prop_schemaZ anyof_schemar"r"r#rsb       rrwcCtd|}|r |dSdS)z Get the vertex project id from the url `https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent` z/projects/([^/]+)rbNrWrZgrouprwmatchr"r"r#get_vertex_project_id_from_urlM rcCr)z Get the vertex location from the url `https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent` z/locations/([^/]+)rbNrrr"r"r#get_vertex_location_from_urlWrrcCr)z Get the vertex model id from the url `https://${LOCATION}-aiplatform.googleapis.com/v1/projects/${PROJECT_ID}/locations/${LOCATION}/publishers/google/models/${MODEL_ID}:streamGenerateContent` z/models/([^:]+)rbNrrr"r"r#get_vertex_model_id_from_urlarrrequested_routecCs tdd|d|d|}|S)zS Replace project and location values in the route with the provided values z /projects/[^/]+/locations/[^/]+/rorpr8)rWsub)rrjrgZmodified_router"r"r#%replace_project_and_location_in_routeks rrvc Cst|}d|vr|r|rt|||}|j|dS d}d|vr"d}|dr1d}|ddd}n|d r?d}|d dd}d |||}d||}|j|d}|S) a< Allow user to specify their own project id / location. If missing, use defaults Handle cachedContent scenario - https://github.com/BerriAI/litellm/issues/5460 Constructed Url: POST https://LOCATION-aiplatform.googleapis.com/{version}/projects/PROJECT_ID/locations/LOCATION/cachedContents locations)pathrlZ cachedContentrmz/v1/r8rbz /v1beta1/z{}/projects/{}/locations/{})r*URLr copy_withrdrerR) rvrrgrjZ new_base_urlZvertex_versionZbase_requested_routeZupdated_requested_routeZ updated_urlr"r"r#construct_target_urlzs.      rcCs,ddlm}||dd}|durdSd|vS)z Check if a model is only available in the global region. Args: model: The model name to check Returns: True if the model is only available in global region, False otherwise r)get_supported_regionsrKrMNFrh) litellm.utilsr)r=rZsupported_regionsr"r"r#is_global_only_vertex_models rc@seZdZdeefddZ  ddededee ded ed eed eedefd d Z dd eed eedeefddZ e dd eedeefddZ e dd eedeefddZe dedeefddZdS)VertexAIModelInfor?cCstS)z Factory method to create a token counter for this provider. Returns: Optional TokenCounterInterface implementation for this provider, or None if token counting is not supported. VertexAITokenCounter)rr"r"r#get_token_countersz#VertexAIModelInfo.get_token_counterNrr=messagesoptional_paramsr>api_keyrBcCtdN&Vertex AI models are not supported yetNotImplementedError)rrr=rrr>rrBr"r"r#validate_environments z&VertexAIModelInfo.validate_environmentcCr)zF Returns a list of models supported by this provider. rr)rrrBr"r"r# get_modelsszVertexAIModelInfo.get_modelscCrrr)rr"r"r# get_api_keyszVertexAIModelInfo.get_api_keycCrrr)rBr"r"r# get_api_baseszVertexAIModelInfo.get_api_basecCr)a2 Returns the base model name from the given model name. Some providers like bedrock - can receive model=`invoke/anthropic.claude-3-opus-20240229-v1:0` or `converse/anthropic.claude-3-opus-20240229-v1:0` This function will return `anthropic.claude-3-opus-20240229-v1:0` rrrCr"r"r#get_base_modelsz VertexAIModelInfo.get_base_model)NNr$)r%r&r'rrrrr)rrrr staticmethodrrrr"r"r"r#rsP    rc@seZdZdZ ddeedefddZ    dded eee ee fd eee ee fd ee ee fd ed eee ee fdee dee fddZ dS)rz;Token counter implementation for Google AI Studio provider.NrJr?cCsddlm}||jjkS)Nr) LlmProviders)litellm.types.utilsrZ VERTEX_AIr9)rrJrr"r"r#should_use_token_counting_apis  z2VertexAITokenCounter.should_use_token_counting_apira model_to_usercontents deployment request_modeltoolssystemcs>ddl}ddlm} |pi}||di} | |rl| } | dp)| d} | dp3| d} | dp:| } | d pD| d }| j||pKg| | | |d IdH}|durjt|d d|||d d|dSdSddlm }||d}| ||j di| IdH}|durt|dd|||d d|dSdS)Nrr@r>rjZvertex_ai_projectrgZvertex_ai_locationZvertex_count_tokens_locationvertex_credentialsZvertex_ai_credentials)r=rr>rjrgrZ input_tokensZtokenizer_usedra)Z total_tokensrZ model_usedZtokenizer_typeoriginal_responser)r=rZ totalTokensr") copyrDrArrErGr`rZ+litellm.llms.vertex_ai.count_tokens.handlerrupdateZ acount_tokens)rrrrrrrrrrAZcount_tokens_params_requestZpartner_models_handlerrjrgrrrZcount_tokens_paramsr"r"r#r`s~             z!VertexAITokenCounter.count_tokensr$)NraNN) r%r&r'r0rr)rYrrrrrr`r"r"r"r#rs:   rr$)r)F)NrWrrrrtypingrrrrrr r r r r*rNZlitellm._loggingr Zlitellm.constantsrZ8litellm.litellm_core_utils.prompt_templates.common_utilsrZ litellm.llms.base_llm.base_utilsrrZ)litellm.llms.base_llm.chat.transformationrZlitellm.types.llms.openairZlitellm.types.llms.vertex_airrrrrrrrr)r,rcrrIrYrUrVr[Zall_gemini_url_modesrfrirxrrrrrrrrrr(rrrrrrrrrrrrrrrrr"r"r"r#s  ,         G  $    #  :  3  33"       ./! G     54