o
ưi6 @ s d dl Z d dlmZ d dlmZmZmZmZ d dlZd dl Z d dl
m Z d dl
mZ d dlmZmZ d dlmZmZ d dlmZ d d lmZmZ G d
d d eZd ed
e de de de de f ddZ G dd dZ dS ) N)partial)CallableListOptionalUnion)Logging)track_llm_api_timing)
custom_promptprompt_factory)AsyncHTTPHandlerget_async_httpx_client) TextChoices)CustomStreamWrapperTextCompletionResponsec s8 e Zd Z ddeej deej f fdd
Z ZS ) TextCompletionCodestralErrorNrequestresponsec s` || _ || _|d ur|| _ntjddd| _|d ur || _n tj|| jd| _t | j d S )NPOSTz8https://docs.codestral.com/user-guide/inference/rest_api)methodurl) status_coder ) r messager httpxRequestr Responsesuper__init__)selfr r r r __class__ `/home/app/Keep/.python/lib/python3.10/site-packages/litellm/llms/codestral/completion/handler.pyr s z%TextCompletionCodestralError.__init__)NN) __name__
__module__ __qualname__r r r r r
__classcell__r r r r! r s r clientapi_baseheadersdatamodelmessagesc sV | j |||ddI d H }|jdkrt|j|jd| }|j|d|d|id |S )NTr( r) stream r r complete_input_dictinputapi_keyoriginal_responseadditional_args)postr r textZ aiter_lines post_call) r& r' r( r) r* r+ logging_objr completion_streamr r r! make_call5 s
r< c sf e Zd Zd# fdd Zdee dedefddZd efd
d Zd ed
e j
de de de
dededeeef dede fddZdddi fd edededede dedededeee jf dedee ef fdd Ze ddi fd ededede dedededeee jf de fd d Ze dddi fd ededede dededeee jf defd d Zd!d" Z ZS )$CodestralTextCompletionreturnNc s t d S N)r r )r r r r! r R s z CodestralTextCompletion.__init__r4 user_headersc C sB |d u rt ddd|d}|d ur t|tr i ||}|S )NzVMissing CODESTRAL_API_Key - Please add CODESTRAL_API_Key to your environment variableszapplication/jsonz Bearer {})z content-type
Authorization)
ValueErrorformat
isinstancedict)r r4 r@ r( r r r! _validate_environmentU s z-CodestralTextCompletion._validate_environmentgenerated_textc C sh g d}|D ]+}| |r||dd}||r1|ddd |ddd ddddd }q|S )z
Parse the output text to remove any special characters. In our current approach we just check for ChatML tokens.
Initial issue that prompted this - https://github.com/BerriAI/litellm/issues/763
)z
<|assistant|>z
<|system|>z<|user|>zzr0 N)strip
startswithreplaceendswith)r rG Zchat_template_tokenstokenr r r!
output_parserf s
,z%CodestralTextCompletion.output_parserr* r model_responser- r: optional_paramsr) r+ c
C s |j | ||jd|id |
d|j |jdkr#tt|j|jdz| } W n ty7 t|jddw | dg }
g }|
D ]+}d }d}d }d }|d i }tj j
|d
|d |d |d
d}| | qBtj | d|| d| d| dd| dd}|S )Nr1 r2 z#codestral api: raw model_response: r. )r r i choicesr r
finish_reasonindexcontentlogprobs)rS rT r8 rV idcreatedr* usageFobject)rW rR rX r* rY r- rZ )
r9 r8 r r strjson Exceptiongetlitellmutilsr
appendr )r r* r rP r- r: rQ r4 r) r+
print_verboseencodingZcompletion_responseZ_original_choicesZ_choiceschoiceZ_finish_reason_index_textZ _logprobsZ_choice_messageZ_choice _responser r r! process_text_completion_responsez sT
z8CodestralTextCompletion.process_text_completion_responser' custom_prompt_dictrb timeoutr( c C s | ||}|
dd du r|}n|pd}||v r,|| }t|d |d |d |d}nt||d}tj }| D ] \}}||
vrG||
|< q;|
d d
}||d |
}|}| j|||||| d d
| du r|du r| j ||||||||| |
|
||| dS | j
||||||||| |
d
|
||| dS |du rtj j ||t
||d}t| |d| d}|S tj j ||t
|d}| j||||
d d
| |
|||||d S )NZcustom_endpointTz/https://codestral.mistral.ai/v1/fim/completionsZrolesinitial_prompt_valuefinal_prompt_value)Z role_dictrk rl r+ )r* r+ r- F)r* prompt)r1 r( r' acompletion)r3 r4 r6 )r* r+ r) r' rP rb rc r4 r: rQ litellm_params logger_fnr( rj )r* r+ r) r' rP rb rc r4 r: rQ r- ro rp r( rj r, Z codestral)custom_llm_providerr: )r r( r) ) r* r rP r- r: rQ r4 r) r+ rb rc )rF popr r
r_ Z CodestralTextCompletionConfigZ
get_configitemsZpre_callasync_streamingasync_completionZmodule_level_clientr7 r\ dumpsr
iter_linesrh r^ )r r* r+ r' ri rP rb rc r4 r: rQ rj rn ro rp r( Zcompletion_urlZmodel_prompt_detailsrm configkvr- r) Z
input_textr rg r r r!
completion s
z"CodestralTextCompletion.completionc
s t tjjd| id}z|j||t|
dI d H }W n0 tjy5 } z
t |j
j d |j
j
dd }~w tyK } z t dd t|dd }~ww | j|||| |||
||| |d S ) Nrj )Z llm_providerparams)r( r) zHTTPStatusError - {}r/ i z{}) r* r rP r- r: r4 r) r+ rb rQ rc )r r_ Z LlmProvidersZTEXT_COMPLETION_CODESTRALr7 r\ rv r ZHTTPStatusErrorr r r rC r8 r] r[ rh )r r* r+ r' rP rb rc r4 r: r- r) rQ rj ro rp r( Z
async_handlerr er r r! ru Q sD z(CodestralTextCompletion.async_completionc s8 d| d< t d tt||t| |||d|d|d}|S )NTr- )r' r( r) r* r+ r: ztext-completion-codestral)r; r< r* rq r: )r r r< r\ rv )r r* r+ r' rP rb rc r4 r: r) rj rQ ro rp r( Z
streamwrapperr r r! rt s$ z'CodestralTextCompletion.async_streamingc O s d S r? r )r argskwargsr r r! embedding s z!CodestralTextCompletion.embedding)r> N)r" r# r$ r r r[ rE rF rO r r r boolLiteLLMLoggingr listrh r floatTimeoutr r{ r ru rt r r% r r r r! r= Q s
V
1
$r= )!r\ functoolsr typingr r r r r r_ Z*litellm.litellm_core_utils.litellm_loggingr r Z(litellm.litellm_core_utils.logging_utilsr Z3litellm.litellm_core_utils.prompt_templates.factoryr r
Z&litellm.llms.custom_httpx.http_handlerr r Zlitellm.types.utilsr
Z
litellm.utilsr r r] r r[ rE r r< r= r r r r! s4