o
    ưiW                     @   s   d dl Z d dlmZ d dlZd dlmZmZ d dlmZm	Z	 da
G dd deZdefdd	Zi ddfded
edededef
ddZdi fded
efddZdd ZdS )    N)Callable)custom_promptprompt_factory)ModelResponseUsagec                       s   e Zd Z fddZ  ZS )	VLLMErrorc                    s@   || _ || _tjddd| _tj|| jd| _t | j d S )NPOSTzhttp://0.0.0.0:8000)methodurl)status_coderequest)	r   messagehttpxRequestr   Responseresponsesuper__init__)selfr   r   	__class__ [/home/app/Keep/.python/lib/python3.10/site-packages/litellm/llms/vllm/completion/handler.pyr      s   zVLLMError.__init__)__name__
__module____qualname__r   __classcell__r   r   r   r   r      s    r   modelc              
   C   sV   zddl m}m} td u r|| dat|fW S  ty* } ztdt|dd }~ww )Nr   )LLMSamplingParamsr   r   r   )Zvllmr   r   llm	Exceptionr   str)r   r   r   er   r   r   validate_environment   s   

r&   messagesmodel_responseprint_verboseoptional_paramsc
              
   C   sh  z	t | d\a}
W n ty } ztdt|dd }~ww |
di |}| |v r<||  }t|d |d |d |d}nt| |d}|j|d	d
|id trUt||}ntdddd|v ri|d du rit	|S |j
|d	|d
|id |d|  |d jd j|jd j_t|d j}t|d jd j}tt |_| |_t|||| d}t|d| |S )Nr    r   r!   rolesinitial_prompt_valuefinal_prompt_valueZ	role_dictr,   r-   r'   r   r'    Zcomplete_input_dict)inputapi_keyadditional_args/Need to pass in a model name to initialize vllmstreamT)r1   r2   original_responser3   zraw model_response: prompt_tokenscompletion_tokensZtotal_tokensusager   )r&   r"   r#   r   r$   r   r   Zpre_callgenerateiterZ	post_calloutputstextchoicesr   contentlenprompt_token_ids	token_idsinttimecreatedr   r   setattr)r   r'   r(   r)   encodingZlogging_objr*   custom_prompt_dictZlitellm_paramsZ	logger_fnr   r%   sampling_paramsmodel_prompt_detailspromptr=   r8   r9   r:   r   r   r   
completion'   s^   rM   c              
   C   sV  z	t | d\}}W n ty } z
t|}td|dd}~ww |di |}g }	| |v rJ||  }
|D ]}t|
d |
d |
d |d}|	| q3n|D ]}t| |d	}|	| qL|rc||	|}ntdd
dg }|D ];}t }|j	d j
|jd j_t|j}t|j	d j}tt |_| |_t|||| d}t|d| || qm|S )a  
    Example usage:
    import litellm
    import os
    from litellm import batch_completion


    responses = batch_completion(
        model="vllm/facebook/opt-125m",
        messages = [
            [
                {
                    "role": "user",
                    "content": "good morning? "
                }
            ],
            [
                {
                    "role": "user",
                    "content": "what's the time? "
                }
            ]
        ]
    )
    r    r   r!   Nr+   r,   r-   r.   r/   r4   r7   r:   r   )r&   r#   r$   r   r   appendr   r;   r   r=   r>   r?   r   r@   rA   rB   rC   rD   rE   rF   r   r   rG   )r   r'   r*   rI   r"   r   r%   Z	error_strrJ   ZpromptsrK   r   rL   r=   Zfinal_outputsoutputr(   r8   r9   r:   r   r   r   batch_completionsq   sX   	
rP   c                   C   s   d S )Nr   r   r   r   r   	embedding   s   rQ   )rE   typingr   r   Z3litellm.litellm_core_utils.prompt_templates.factoryr   r   Zlitellm.utilsr   r   r"   r#   r   r$   r&   listdictrM   rP   rQ   r   r   r   r   <module>   s:    
K
P