o
ưi@ @ s d dl Z d dlZd dlZd dlmZ d dlmZmZmZ d dl Z d dl
Z
d dl Z
d dl Z
d dl
mZ d dlmZmZ d dlmZmZ d dlmZ d dlmZmZmZmZm Z d d
l m Z d ed e d
e de de de!deee"e j#f fddZ$G dd dZ%dS ) N)partial)CallableOptionalUnion)map_finish_reason)
custom_promptprompt_factory)AsyncHTTPHandlerget_async_httpx_client)LiteLLMLoggingBaseClass)ChoicesCustomStreamWrapperMessage
ModelResponseUsage )PredibaseErrorclientapi_baseheadersdatamodelmessagestimeoutc
sX | j |||d|dI d H }|jdkrt|j|jd| } |j|d| d|id | S )NTr r streamr status_codemessage complete_input_dictinputapi_keyoriginal_responseadditional_args)postr r textZ aiter_lines post_call)
r r r r r r logging_objr responsecompletion_stream r- Z/home/app/Keep/.python/lib/python3.10/site-packages/litellm/llms/predibase/chat/handler.py make_call s
r/ c sF e Zd Zd" fdd ZdefddZded ejd
ed e d e
d
e dede e ef de
defddZddi fdede
dede d
ededed
e de dede eejf de de eef fddZddi fdede
ded
edede d
e de eejf defd d Zdddi fdede
ded
edede de eejf defd d Zd d! Z ZS )#PredibaseChatCompletionreturnNc s t d S N)super__init__)self __class__r- r. r4 < s z PredibaseChatCompletion.__init__generated_textc C sh g d}|D ]+}| |r||dd}||r1|ddd |ddd ddddd }q|S )z
Parse the output text to remove any special characters. In our current approach we just check for ChatML tokens.
Initial issue that prompted this - https://github.com/BerriAI/litellm/issues/763
)z
<|assistant|>z
<|system|>z<|user|>zzr N)strip
startswithreplaceendswith)r5 r8 Zchat_template_tokenstokenr- r- r.
output_parser? s
,z%PredibaseChatCompletion.output_parserr r+ model_responser r* optional_paramsr$ r r c C s* |j | ||jd|id |
d|j z| } W n ty( t|jddw d| v r8tt| d |jdt| tsFtdd| dd | vrStdd
| dt | d d krg|
| d |j d j _
d | v rd
| d v rt| d d |j d _d }
| d d
D ]}|d d ur|
|d 7 }
qt|j d j d|
d|v r |d dkr d | v r d| d v r g }t| d d D ]D\}}d }
|d
D ]}|d d ur|
|d 7 }
qt |d d krt|
|d |
d}ntd d}tt|d |d |d}|| q|j | d }ztj| d}W n
ty Y nw |d d d dd}|d urXt |d krXd }zt | |d d d dd}W n tyW Y nw d }|| }tt |_ ||_ t |||d }||_ |j }i }|! D ]\}} |"d r| |d #|< q{||j$d < |S ) Nr! r" zraw model_response: i )r r errorz,'completion_response' is not a dictionary - r r8 z4'generated_text' is not a key response dictionary - r detailstokens
finish_reasonZlogprobZ_logprobZbest_ofr9 Zbest_of_sequences)contentZlogprobs)rG )rF indexr )r choicesr rG r )
prompt_tokenscompletion_tokens total_tokenszx-zllm_provider-{}Zadditional_headers)%r) r( json Exceptionr strr
isinstancedictlenr@ rI r rG r rF setattr enumerater r appendextendlitellmZ
token_countergetencodeinttimecreatedr r usager itemsr< formatZ_hidden_params) r5 r r+ rA r r* rB r$ r r
print_verboseencodingZcompletion_responseZ sum_logprobr? Z choices_listidxitemZ message_objZ
choice_objrJ Z output_textrK rL r] Zpredibase_headersresponse_headerskvr- r- r. process_responseS s
z(PredibaseChatCompletion.process_responser custom_prompt_dictr` litellm_params tenant_idr r c C s t j||||
|| d}d}d}d}d|v r|}n|r |}n dtjv r*tdd}| d| d| }|
dd d
u rA|d 7 }n|d 7 }||v r\|| }t|d
|d |d |d}nt||d}t j }|
D ] \}}||
vrw||
|< qk|
dd }||
d}|}| j ||||||dd |d
u r|d
u r| j
||||||||| |
| |||
dS | j||||||||| |
d | |||
dS |d
u rt jj||t|||
d}t| |d| d} | S t jj||t||
d}| j||||
dd | |
|||||d S ) N)r$ r r rB r ri r z!https://serving.app.predibase.comhttpsZPREDIBASE_API_BASE/z/deployments/v2/llms/r FTz/generate_streamz /generateZrolesinitial_prompt_valuefinal_prompt_value)Z role_dictrm rn r )r r )Zinputs
parameters)r! r r acompletion)r# r$ r& )r r r r rA r` ra r$ r* rB ri logger_fnr r )r r r r rA r` ra r$ r* rB r ri rq r r r predibase)custom_llm_providerr* )urlr r r ) r r+ rA r r* rB r$ r r r` ra )rW ZPredibaseConfigZvalidate_environmentosenvirongetenvrX r r Z
get_configr^ popZpre_callasync_streamingasync_completionZmodule_level_clientr' rM dumpsr
iter_linesrg ) r5 r r r rh rA r` ra r$ r* rB ri rj r rp rq r Zcompletion_urlZ
input_textbase_urlZmodel_prompt_detailspromptconfigre rf r r r+ _responser- r- r.
completion s
z"PredibaseChatCompletion.completionc
s t tjjd| id}z|j||t|
dI d H }W n@ tjy8 } zt |j
j d |j
j |j
j
dd }~w ty[ } ztjD ] }t||rK|qBt dd t|dd }~ww | j|||| |||
||| |d S ) Nr )Z llm_providerparams)r r z;HTTPStatusError - received status_code={}, error_message={}r i z{}) r r+ rA r r* r$ r r r` rB ra )r
rW Z LlmProvidersZ PREDIBASEr' rM r{ httpxZHTTPStatusErrorr r+ r r_ r( rN ZLITELLM_EXCEPTION_TYPESrP rO rg )r5 r r r rA r` ra r$ r* r r rB r ri rq r Z
async_handlerr+ e exceptionr- r- r. rz | sP
z(PredibaseChatCompletion.async_completionc s: d| d< t d tt||t| ||||
d|d|d}|S )NTr )r r r r r r* r rr )r, r/ r rs r* )r
r r/ rM r{ )r5 r r r rA r` ra r$ r* r r rB ri rq r Z
streamwrapperr- r- r. ry s&
z'PredibaseChatCompletion.async_streamingc O s d S r2 r- )r5 argskwargsr- r- r. embedding s z!PredibaseChatCompletion.embedding)r1 N)__name__
__module__ __qualname__r4 rO r@ r Responser boolr rQ r listrg r floatTimeoutr
r rz ry r
__classcell__r- r- r6 r. r0 ; s
0
B
%r0 )&rM ru r[ functoolsr typingr r r r rW Zlitellm.litellm_core_utilsZ*litellm.litellm_core_utils.litellm_loggingZ'litellm.litellm_core_utils.core_helpersr Z3litellm.litellm_core_utils.prompt_templates.factoryr r Z&litellm.llms.custom_httpx.http_handlerr r
Zlitellm.types.utilsr Z
litellm.utilsr r
r r r Z common_utilsr rO rQ r r r r/ r0 r- r- r- r. s>