o
    ưi!                    @   sr  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZmZ d dlmZ d dlZd dlZd dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
l m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z'm(Z( d dl)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/T ddl0T ddl1m2Z2m3Z3 ddl4m5Z5 erd dl6m7Z8 e8Z7neZ7G dd deZ9dS )    N)	timedelta)TYPE_CHECKINGAnyDictListLiteralOptionalTupleUnion)APIError)verbose_loggerverbose_proxy_logger)	DualCache)HOURS_IN_A_DAY)CustomBatchLogger)get_budget_alert_type)AlertingHangingRequestCheck)duration_in_seconds)_add_key_name_and_team_to_alert)get_async_httpx_clienthttpxSpecialProvider)	AlertTypeCallInfoLitellm_EntityTypeVirtualKeyEventWebhookEvent)*      )send_to_webhooksquash_payloads) process_slack_alerting_variables)Routerc                       s  e Zd ZdZddg edi ddfdee dee dee dee	 dee
e	eee ef f  dee d	ee
eef  f fd
dZ							ddee dee deee	  dee
e	eee ef f  dee
 dee d	ee
eef  fddZdeeeef defddZdee dee fddZdd Zdd Zdd Zdd ZdddZd d! Zd"edefd#d$Zdefd%d&Z	dd'ee fd(d)Z d*ed+efd,d-Z!d.e"d/ d0e#fd1d2Z$d0e#d3ee"d4  d5ede%ee"d4  ef fd6d7Z&d0e#defd8d9Z'd0e#defd:d;Z(d<ee d=ee d>ee d?ee d@ee f
dAdBZ)dCee defdDdEZ*dFe"dG dHe"dI dJedKedLee de+defdMdNZ,dOe-dPeddfdQdRZ.dOe-dPeddfdSdTZ/dUedVedWe0fdXdYZ1dUefdZd[Z2d\e3defd]d^Z4		dd_ed`ee daee fdbdcZ5d\e3defdddeZ6d\e3dFedefdfdgZ7			ddhedie"dj dFe	dked0ee3 dlee dLee fdmdnZ8dodp Z9dqdr Z:dsdt Z;dudv Z<dwdx Z=defdydzZ>ddee0 fd{d|Z?	}dd~efddZ@dd ZAdd ZBdeCdFe	defddZDd'ee defddZE  ZFS )SlackAlertingz(
    Class for sending Slack Alerts
    Ninternal_usage_cachealerting_thresholdalertingalert_typesalert_to_webhook_urldefault_webhook_urlalert_type_configc	                    s   |d u rd}|| _ || _|| _|pt | _ttjd| _t	|d| _
d| _tdi || _|| _t | _d| _t| d| _i | _|r]| D ]\}
}t|trWtdi |n|| j|
< qGi | _t | _t jdi |	d| ji d S )Ni,  )Zllm_providerr(   F)Zslack_alerting_object
flush_lock )r%   r&   r'   r   r$   r   r   ZLoggingCallbackasync_http_handlerr!   r(   
is_runningSlackAlertingArgsalerting_argsr)   asyncioLockr,   periodic_startedr   hanging_request_checkr*   items
isinstancedictAlertTypeConfigdigest_bucketsdigest_locksuper__init__)selfr$   r%   r&   r'   r(   r1   r)   r*   kwargskeyval	__class__r-   h/home/app/Keep/.python/lib/python3.10/site-packages/litellm/integrations/SlackAlerting/slack_alerting.pyr=   <   s6   
$
 zSlackAlerting.__init__r1   
llm_routerc                 C   s   |d ur|| _ t|   d| _|d ur|| _|d ur|| _|d ur8tdi || _| js8t|   d| _|d urV|	 D ]\}}	t
|	trPtdi |	n|	| j|< q@|d urs| jd u rft|d| _nt|dpli }
| j|
 |d ur||| _d S d S )NTr+   r-   )r&   r2   create_taskperiodic_flushr4   r%   r'   r0   r1   r6   r7   r8   r9   r*   r(   r!   updaterE   )r>   r&   r%   r'   r(   r1   rE   r*   r@   rA   Z_new_valuesr-   r-   rD   update_valuesh   s<   
$


zSlackAlerting.update_valuesoutage_valuereturnc                 C   s2   t |}d|v rt|d trt|d |d< |S )z
        Helper method to prepare outage value for Redis caching.
        Converts set objects to lists for JSON serialization.
        deployment_ids)r8   r7   setlist)r>   rJ   cache_valuer-   r-   rD   _prepare_outage_value_for_cache   s   
z-SlackAlerting._prepare_outage_value_for_cachec                 C   s(   |rt |dtrt|d |d< |S )z
        Helper method to restore outage value after retrieving from cache.
        Converts list objects back to sets for proper handling.
        rL   )r7   getrN   rM   )r>   rJ   r-   r-   rD    _restore_outage_value_from_cache   s   z.SlackAlerting._restore_outage_value_from_cachec                       d S Nr-   r>   r-   r-   rD   deployment_in_cooldown      z$SlackAlerting.deployment_in_cooldownc                    rS   rT   r-   rU   r-   r-   rD    deployment_removed_from_cooldown   rW   z.SlackAlerting.deployment_removed_from_cooldownc                 C   s   t tS rT   )rN   r   rU   r-   r-   rD   _all_possible_alert_types   s   z'SlackAlerting._all_possible_alert_typesc              
   C   s   z:|| }|  }|di }|dd}tj||d}|dd }	|	d u r,|dd }	t|	d d }
||||
fW S  tyH } z|d }~ww )Nlitellm_paramsmodel r[   Zoptional_paramsmessagesinputd   )total_secondsrQ   litellmget_api_basestr	Exception)r>   r?   
start_timeend_timeZtime_differencetime_difference_floatrZ   r[   api_baser^   Z	_messageser-   r-   rD   )_response_taking_too_long_callback_helper   s   z7SlackAlerting._response_taking_too_long_callback_helperc                 C   s   |d u rd S d|v rZ	 d}|d }t |dkrd S d }zt| dd d}t|}W n	 ty4   Y nw |d u r;d S | D ]\}}|d| dt|d	 d
7 }q?d| d }|S d S )NZ_latency_per_deploymentr\   r   c                 S   s   | d S )Nr   r-   )xr-   r-   rD   <lambda>       zBSlackAlerting._get_deployment_latencies_to_alert.<locals>.<lambda>r@   
: r   sz```)lensortedr6   r8   re   round)r>   metadataZ_message_to_sendZ_deployment_latencies_deployment_latency_mapri   Zlatencyr-   r-   rD   "_get_deployment_latencies_to_alert   s0   z0SlackAlerting._get_deployment_latencies_to_alertc                    s  | j d u s| jd u rd S | j|||d\}}}}tjstjr!d}d| d| d| d}	dt|d d	| j d
}
i }|| jkr|d urtd|v rtd|d v rt|d d }t|	|d}	| j	|d}|d url|	d| 7 }	d|v rt|d }| j
|
|	 dtj|||dI d H  d S d S )N)r?   rf   rg   z>Message not logged. litellm.redact_messages_in_exceptions=Truez
Request Model: `z`
API Base: `z`
Messages: ``z`Responses are slow - r   z&s response time > Alerting threshold: zs`rZ   rv   )request_inforv   )rv   z 
Available Deployment Latencies
alerting_metadataLow)messagelevel
alert_typer{   request_modelri   )r&   r'   rk   rb   Zturn_off_message_loggingZredact_messages_in_exceptionsru   r%   r   rx   
send_alertr   Zllm_too_slow)r>   r?   Zcompletion_responserf   rg   rh   r[   ri   r^   rz   Zslow_messager{   	_metadatarw   r-   r-   rD   !response_taking_too_long_callback   sX   
z/SlackAlerting.response_taking_too_long_callbackdeployment_metricsc                    s   d}z:|j r| jjd|jtjjdddI dH  |d7 }|jdur;| jjd|jtj	j|jddI dH  |d7 }|W S  t
yG   Y dS w )ax  
        Store the perf by deployment in cache
        - Number of failed requests per deployment
        - Latency / output tokens per deployment

        'deployment_id:daily_metrics:failed_requests'
        'deployment_id:daily_metrics:latency_per_output_token'

        Returns
            int - count of metrics set (1 - if just latency, 2 - if failed + latency)
        r   {}:{}r   N)r@   valueparent_otel_span)failed_requestr$   Zasync_increment_cacheformatidSlackAlertingCacheKeysfailed_requests_keyr   latency_per_output_tokenlatency_keyre   )r>   r   
return_valr-   r-   rD   async_update_daily_reports.  s4   	

z(SlackAlerting.async_update_daily_reportsc                    s@  |  }dd |D }dd |D }|| }| jj|dI dH }|du r'dS d}|D ]}|dur9|dkr9d} nq+|r>dS |dt| }	|t|d }
d  fd	d|	D tttfd
ddddd }fdd|D }d  fdd|
D tttfdddddd }fdd|D }dt  d}|d7 }|s|d7 }tt|D ]Q}|||  dd }||}t	|t
r|d dd}n dS tj||dur|d ni d}|du rd}||  }|d|d  d| d| d| d 	7 }q|d!7 }|s|d7 }tt|D ]O}|||  dd }||}|dur7|d dd}nd}tj||durE|d ni d}t||  d"}|d|d  d| d#| d$| d%	7 }qd&d |D }d'd |D }|| }| jj|d(I dH  |d)t | jj  d*7 }| j|d+tji d,I dH  dS )-a  
        Send a daily report on:
        - Top 5 deployments with most failed requests
        - Top 5 slowest deployments (normalized by latency/output tokens)

        Get the value from redis cache (if available) or in-memory and send it

        Cleanup:
        - reset values in cache -> prevent memory leak

        Returns:
            True -> if successfuly sent
            False -> if not sent
        c                 S      g | ]
}d  |tjjqS r   )r   r   r   r   .0r   r-   r-   rD   
<listcomp>o      z4SlackAlerting.send_daily_reports.<locals>.<listcomp>c                 S   r   r   )r   r   r   r   r   r-   r-   rD   r   s  s    )keysNFTr   c                       g | ]
}|d ur
|n qS rT   r-   r   r   placeholder_valuer-   rD   r     r   c                        |  S rT   r-   ireplaced_failed_valuesr-   rD   rm     rn   z2SlackAlerting.send_daily_reports.<locals>.<lambda>)r@   reverse   c                       g | ]
} | d kr|qS r   r-   r   indexr   r-   rD   r         c                    r   rT   r-   r   r   r-   rD   r     r   c                    r   rT   r-   r   replaced_slowest_valuesr-   rD   rm     rn   c                    r   r   r-   r   r   r-   rD   r     r   z	

Time: `u(   `s
Here are today's key metrics 📈: 

u7   

*❗️ Top Deployments with Most Failed Requests:*

z	None
:rZ   r[   r\   r]   	r   z. Deployment: `z`, Failed Requests: `z`,  API Base: ``
u#   

*😅 Top Slowest Deployments:*

   z`, Latency per output token: `zs/token`,  API Base: ``

c                 S      g | ]}|d fqS r   r-   r   r@   r-   r-   rD   r         c                 S   r   r   r-   r   r-   r-   rD   r     r   )Z
cache_listz

Next Run is at: `z`sr|   r}   r~   r   r{   )Zget_model_idsr$   Zasync_batch_get_cachers   rt   rangetimesplitZget_model_infor7   r8   rQ   rb   rc   ru   Zasync_set_cache_pipeliner1   daily_report_frequencyr   r   daily_reports)r>   routeridsZfailed_request_keysZlatency_keysZcombined_metrics_keysZcombined_metrics_valuesZall_nonerA   Zfailed_request_valuesZlatency_valuesZtop_5_failedZtop_5_slowestr}   r   r@   Z_deploymentZdeployment_nameri   r   Zlatency_cache_keysZfailed_request_cache_keysZcombined_metrics_cache_keysr-   )r   r   r   rD   send_daily_reports\  s   










(

*z SlackAlerting.send_daily_reportsrequest_datac                    sB   | j d u s| jd u rd S tj| jvrd S | jj|dI d H  d S )N)r   )r&   r'   r   Zllm_requests_hangingr5   Z$add_request_to_hanging_request_check)r>   r   r-   r-   rD   response_taking_too_long  s   z&SlackAlerting.response_taking_too_longerror_messagefailing_modelc                    s   | j du s| jdu rdS d| jvrdS | j}d| }d|}|j|dI dH }|du rI| j|dtji dI dH  |j|d| j	j
d	I dH  dS dS )
z
        Raise alert when tracking failed for specific model

        Args:
            error_message (str): Error message
            failing_model (str): Model that failed tracking
        Nfailed_tracking_spendzFailed Tracking Cost for z budget_alerts:failed_tracking:{}ro   Highr   SENTr@   r   ttl)r&   r'   r$   r   async_get_cacher   r   r   async_set_cacher1   budget_alert_ttl)r>   r   r   _cacher}   
_cache_keyresultr-   r-   rD   failed_tracking_alert   s,   

z#SlackAlerting.failed_tracking_alerttype)	Ztoken_budgetZuser_budgetsoft_budgetZmax_budget_alertZteam_budgetZorganization_budgetZproxy_budgetprojected_limit_exceededZproject_budget	user_infoc                    s0  | j }| jdu s| jdu rdS d| jvrdS t|}||}|jdd}| |}| }|dkr5dnd}	d}
|jdu rE|j	du rEdS | j
|	||d\}	}|	dur|jdurd|	|}|j|dI dH }|du rtd|	|d	|}
| j|d
 | dtj|
i dI dH  |j|d| jjdI dH  dS dS )z
        Send a budget alert on slack or webhook

        Args:
            type: The type of budget alert to send
            user_info: The user info to send the alert for
        Nbudget_alertsTZexclude_noner   )eventr   event_messagezbudget_alerts:{}:{}ro   )r   r   

r   )r}   r~   r   r   r{   r   r   r-   )r$   r&   r'   r   Zget_id
model_dump_get_user_info_strZget_event_message
max_budgetr   _get_event_and_event_messageevent_groupr   r   r   r   r   r   r   r1   r   )r>   r   r   r   Zbudget_alert_classZ_idZuser_info_jsonZuser_info_strr   r   webhook_eventr   r   r-   r-   rD   r     sZ   




zSlackAlerting.budget_alertsr   )budget_crossedthreshold_crossedsoft_budget_crossedr   r   c                 C   s   | j |d}|jdur|j|jkrd}|d|j d7 }|jdurN|j|jkr6d}|d|j d7 }||fS |tkrDd}|d	7 }||fS |tkrNd}|d
7 }||fS )z
        Get the event and event message for a budget alert

        This will append any new information to the event_message

        Handles Max Budget and Soft Budget Alerts
        )r   Nr   zTotal Soft Budget:`ry   r   zBudget Crossed
 Total Budget:`r   z5% Threshold Crossed z15% Threshold Crossed)_get_percent_of_max_budget_leftr   spendr   Z"SLACK_ALERTING_THRESHOLD_5_PERCENTZ#SLACK_ALERTING_THRESHOLD_15_PERCENT)r>   r   r   r   percent_leftr-   r-   rD   r   {  s(   


z*SlackAlerting._get_event_and_event_messagec                 C   s8   d}|j }|j}|du r|S |dkr|S || | }|S )z@
        Get the percent of the max budget that is left
        g        Nr   )r   r   )r>   r   r   Zcurrent_spendr   r-   r-   rD   r     s   z-SlackAlerting._get_percent_of_max_budget_leftc                 C   sV   |j dd}|d d}| D ]\}}t|tr|j}|d| d| d7 }q|S )z>
        Create a standard message for a budget alert
        Tr   tokenr\   r   :* `r   )r   popr6   r7   r   r   )r>   r   Z_all_fields_as_dictmsgkvr-   r-   rD   r     s   

z SlackAlerting._get_user_info_strr   	key_aliasend_user_idresponse_costr   c                    s   | j d ur8d| j v r:|d ur<|d ur>|d ur@t||||d d d |d d dtjd||d}| j|dI d H  d S d S d S d S d S d S )NwebhookZspend_trackedz-Customer spend tracked. Customer={}, spend={})r   r   r   Zcustomer_iduser_idteam_id
user_emailr   Zprojected_exceeded_dateZprojected_spendr   r   r   r   )r&   r   r   ZEND_USERr   send_webhook_alert)r>   r   r   r   r   r   r   r-   r-   rD   customer_spend_alert  s:   
	
z"SlackAlerting.customer_spend_alertalertsc                 C   s   dddd}|D ]$}|dkr|d  d7  < q|dkr$|d  d7  < q|d  d7  < qd	}|  D ]\}}|dkrC|d
||7 }q3|S )z
        Parameters:
        - alerts: List[int] -> list of error codes (either 408 or 500+)

        Returns:
        - str -> formatted string. This is an alert message, giving a human-friendly description of the errors.
        r   )Timeout Errors
API ErrorsUnknown Errors  r   r     r   r   r\   z
{}: {}
)r6   r   )r>   r   Zerror_breakdownalert	error_msgr@   r   r-   r-   rD   _count_outage_alerts  s   z"SlackAlerting._count_outage_alertsr   )MajorMinorr@   )ModelRegionkey_valproviderri   c                 C   s   | d|d|i}|dur||d< d}|  D ]\}	}
|d|	 d|
 d7 }qd	| d
| d| j|d d dtt |d  d d	S )z!Format an alert message for slackz NameZProviderNzAPI Baserp   r   r   r   u   


*⚠️ z Service Outage*

z

*Errors:*
r   )r   z

*Last Check:* `last_updated_at   z	s ago`


)r6   r   ru   r   )r>   r   r@   r   r   ri   rJ   headersZheaders_strr   r   r-   r-   rD   _outage_alert_msg_factory  s   
z'SlackAlerting._outage_alert_msg_factory	exceptiondeployment_idc                    s  | j du rdS | j j|d}|du rdS |jj}|jj}|du r+tj|d\}}}}|jj}|du r<tjj	||jd}|du rBdS || }| j
j|dI dH }	|	durY| |	}	t|dddu sp|jdkrk|jdk sp| j du rrdS |	du rt }
|
| t||jgd	d	t |
d
}	| |	}| j
j||| jjdI dH  dS t|	d | jjk r|	d |j n	 |	d }
|
| |
|	d< t |	d< |	d d	u rt|	d | jjkrt|
dkr| jdd|d|	|d}| j|dtji dI dH  d|	d< n5|	d d	u r0t|	d | jjkr0t|
dkr0| jdd|d|	|d}| j|dtji dI dH  d|	d< | |	}| j
j||dI dH  dS )z
        Send slack alert if specific provider region is having an outage.

        Track for 408 (Timeout) and >=500 Error codes
        Nmodel_idr[   )custom_llm_providerrZ   ro   status_coder   r   F)Zprovider_region_idr   minor_alert_sentmajor_alert_sentr   rL   r   r   rL   r   r
  r   r   r   r   r@   r   ri   rJ   r   Mediumr   Tr  r   r   r@   r   ) rE   get_deploymentrZ   r[   r  rb   get_llm_providerregion_nameutilsZ_get_model_regionr$   r   rR   getattrr	  rM   addProviderRegionOutageModelr   rP   r   r1   Zregion_outage_alert_ttlrs   max_outage_alert_list_sizeappendminor_outage_alert_thresholdr  r   r   outage_alertsmajor_outage_alert_threshold)r>   r  r  
deploymentr[   r   _r  	cache_keyrJ   Z_deployment_setrO   r   r-   r-   rD   region_outage_alerts-  s   








	



z"SlackAlerting.region_outage_alertsc                    s(  z| j j|dI dH }t|dddu s$|jdkr|jdk s$| jdu r'W dS | jj|d}|du r5W dS |jj}|jj}|du rYzt	j
|d\}}}}W n tyX   d}Y nw t	j||jd	}|du rt||jgd
d
t d}| j j||| jjdI dH  W dS t|d | jjk r|d |j n	 t |d< |d d
u rt|d | jjkr| jdd||||d}	| j|	dtji dI dH  d|d< n,|d d
u rt|d | jjkr| jdd||||d}	| j|	dtji dI dH  d|d< | |}
| j j||
dI dH  W dS  ty   Y dS w )a
  
        Send slack alert if model is badly configured / having an outage (408, 401, 429, >=500).

        key = model_id

        value = {
        - model_id
        - threshold
        - alerts []
        }

        ttl = 1hr
        max_alerts_size = 10
        ro   Nr	  r   r   r  r  r\   r]   F)r  r   r
  r  r   r   r   r   r
  r   r   r  r  r   Tr  r   r   r  )r$   r   r  r	  rE   r  rZ   r[   r  rb   r  re   rc   OutageModelr   r   r1   Zoutage_alert_ttlrs   r  r  r  r  r   r   r  r  rP   )r>   r  r  rJ   r  r[   r   r  ri   r   rO   r-   r-   rD   r    s   


	

	

	
zSlackAlerting.outage_alerts
model_namelitellm_model_namepassed_model_infoc                    s   t |dd }i }d}|d urtj|i }d| d}ntj|i }d}| D ]\}}	|dks6|dkr;d|	}	|| d|	 d	7 }q*d
| d| dtdd d| d| d}
| j|
dt	j
i d}|d urvt|rx|I d H  d S d S d S )N
base_modelr\   zBase Model: `r   Zinput_cost_per_tokenZoutput_cost_per_tokenz{:.8f}rq   rp   u%   
*🚅 New Model Added*
Model Name: `zn

Usage OpenAI Python SDK:
```
import openai
client = openai.OpenAI(
    api_key="your_api_key",
    base_url=PROXY_BASE_URLhttp://0.0.0.0:4000z:
)

response = client.chat.completions.create(
    model="z", # model to send to the proxy
    messages = [
        {
            "role": "user",
            "content": "this is a test request, write a short poem"
        }
    ]
)
```

Model Info: 
```
z
```
r|   r   )r  rb   Z
model_costrQ   r6   r   osgetenvr   r   new_model_addedr2   iscoroutine)r>   r   r!  r"  Zbase_model_from_user
model_infor#  Zmodel_info_strr   r   r}   Z	alert_valr-   r-   rD   model_added_alert?  sB   


zSlackAlerting.model_added_alertc                    rS   rT   r-   )r>   r   r-   r-   rD   model_removed_alertz  rW   z!SlackAlerting.model_removed_alertr   c                    sd   t dd}|du rtd| }ddi}| jj|||dI dH }|jdkr*dS td	|j d
S )z
        Sends structured alert to webhook, if set.

        Currently only implemented for budget alerts

        Returns -> True if sent, False if not.

        Raises Exception
            - if WEBHOOK_URL is not set
        ZWEBHOOK_URLNz$Missing webhook_url from environmentContent-typeapplication/json)urlr  data   Tz#Error sending webhook alert. Error=F)	r&  r'  re   model_dump_jsonr.   postr	  printtext)r>   r   webhook_urlpayloadr  responser-   r-   rD   r   }  s   
z SlackAlerting.send_webhook_alertpremium_useremail_logo_urlemail_support_contactc                    s@   ddl m}m} |dur|d us|d urtd|jj d S )Nr   )CommonProxyErrorsr9  Tz$Trying to Customize Email Alerting
 )litellm.proxy.proxy_serverr<  r9  
ValueErrorZnot_premium_userr   )r>   r9  r:  r;  r<  r-   r-   rD   %_check_if_using_premium_email_feature  s   z3SlackAlerting._check_if_using_premium_email_featurec              
      s.  zddl m} | jd u sd| jvrtd| j W dS ddlm}m} t	dt	dd }t	d	d }| 
|||I d H  |d u rDt}|d u rJt}|j}|j}|j}	|d u rs|	d urs|d urs|jjjd
|	idI d H }
|
d urs|
j}|j}|j}t	dd}d}|d u rtjd| d |jdkrtj||||||d}n<|jdkr|j}d}|d ur|d ur|jjjd|idI d H }|d ur|jpd}tj|||||d}n	tjd| d |  |d| |d}||d |d |d dI d H  W dS  ty } ztd t | W Y d }~dS d }~ww )!Nr   
send_emailemailz;Error sending email alert - 'email' not in self.alerting %sF)r9  prisma_clientSMTP_SENDER_LOGOEMAIL_LOGO_URLEMAIL_SUPPORT_CONTACTr   )wherer$  r%  Alert from LiteLLM Server*Trying to send email alert to no recipientextraZkey_created)r:  recipient_email
key_budget	key_tokenbase_urlr;  Zinternal_user_createdzDefault Teamr   -)r:  rL  	team_namerO  r;  z3Trying to send email alert on unknown webhook event	LiteLLM: tosubjecthtmlrT  rU  rV  Zreceiver_emailrU  rV  TzError sending email alert %s)!litellm.proxy.utilsrA  r&   r   errorr=  r9  rC  r&  r'  r?  LITELLM_LOGO_URLLITELLM_SUPPORT_CONTACTr   r   r   dbZlitellm_usertableZfind_uniquer   r   r8   r   ZKEY_CREATED_EMAIL_TEMPLATEr   r   Zlitellm_teamtable
team_aliasZUSER_INVITED_EMAIL_TEMPLATEr   r2  re   rd   )r>   r   rA  r9  rC  r:  r;  
event_namerL  Zrecipient_user_idZuser_rowrN  rM  rO  email_html_contentr   rQ  Zteam_rowemail_eventrj   r-   r-   rD   &send_key_created_or_user_invited_email  s   



z4SlackAlerting.send_key_created_or_user_invited_emailc              	      s*  ddl m} ddlm} tdtdd}tdd}| |||I dH  |du r-t}|du r3t}|j	}|j
}|j}	|j}
d}|du rNtjd	| d
 |jdkrbd| d|	 d|
 d| d	}|  |d| |d}||d |d |d dI dH  |jtjkrddlm} ||dI dH  dS )z
        Sends structured Email alert to an SMTP server

        Currently only implemented for budget alerts

        Returns -> True if sent, False if not.
        r   )r9  r@  rD  rE  NrF  rH  rI  rJ  r   z
            <img src="zD" alt="LiteLLM Logo" width="150" height="50" />

            <p> Hi ze, <br/>

            Your LLM API usage this month has reached your account's <b> monthly budget of $a   </b> <br /> <br />

            API requests will be rejected until either (a) you increase your monthly budget or (b) your monthly usage resets at the beginning of the next calendar month. <br /> <br />

            If you have any questions, please send an email to zY <br /> <br />

            Best, <br />
            The LiteLLM team <br />
            rR  rS  rT  rU  rV  rW  )send_team_budget_alertr   F)r=  r9  rX  rA  r&  r'  r?  rZ  r[  r   r   r   r   r   rY  r8   r   r2  r   r   ZTEAMZ#litellm.integrations.email_alertingrb  )r>   r   r   r9  rA  r:  r;  r^  rL  Z	user_namer   r_  r`  rb  r-   r-   rD   send_email_alert_using_smtp  s^   


	z)SlackAlerting.send_email_alert_using_smtpr}   r~   )r|   r  r   r{   r   c                    s  | j du rdS | js!| j dur!t| j dkr!t|   d| _d| j v r7|dkr7|dur7| j|dI dH  d| j v rN|dkrN|durN| j||dI dH  d	| j vrUdS || jvr\dS dd
l	m	}	 t
|dt|}
| j|
}|dur|jr| jdur|| jv r| j| }n| jdur| j}ntdd}|du rtd|
 d|pd d|pd }| j4 I dH < |	 }|| jv r| j| d  d7  < || j| d< nt|
|pd|pd||d|||d	| j|< W d  I dH  dS 1 I dH sw   Y  dS |	 d}tdd}t
|d|}d| d}|dks|dkr$|| }n| d| d| d| }|rI| D ]\}}|d| d| d 7 }q8|ra| D ]\}}|d!| d| d 7 }qP|durn|d"| d7 }| jdur|| jv r| j| }n| jdur| j}ntdd}|du rtdd#|i}d$d%i}t|tr|D ]}| j||||d& qn| j||||d& t| j| jkr|  I dH  dS dS )'a  
        Alerting based on thresholds: - https://github.com/BerriAI/litellm/issues/1298

        - Responses taking too long
        - Requests are hanging
        - Calls are failing
        - DB Read/Writes are failing
        - Proxy Close to max budget
        - Key Close to max budget

        Parameters:
            level: str - Low|Medium|High - if calls might fail (Medium) or are failing (High); Currently, no alerts would be 'Low'.
            message: str - what is the alert about
            request_model: Optional[str] - model name for digest grouping
            api_base: Optional[str] - api base for digest grouping
        Nr   Tr   r   r   rB  )r   r   Zslackdatetimer   ZSLACK_WEBHOOK_URLz*Missing SLACK_WEBHOOK_URL from environmentr   r\   countr   	last_time)	r   r   ri   first_messager~   rf  rf   rg  r6  %H:%M:%Sr$  nameAlert type: `ry   r   r(  z	
Level: `z`
Timestamp: ``

Message: r   : `r   z

*Alerting Metadata*: 


Proxy URL: `r5  r-  r.  r/  r  r7  r   ) r&   r4   rs   r2   rF   rG   r   rc  r'   re  r  rd   r*   rQ   digestr(   r)   r&  r'  r>  r;   nowr:   ZDigestEntrystrftimer6   r7   rN   	log_queuer  Z
batch_sizeflush_queue)r>   r}   r~   r   r{   r   r   ri   r?   re  Zalert_type_name_str_atcZ_digest_webhookZ
digest_keyrq  current_time_proxy_base_urlalert_type_nameZalert_type_formattedformatted_messager@   r   Zslack_webhook_urlr7  r  r/  r-   r-   rD   r   Z  s   













	zSlackAlerting.send_alertc                    sJ    j sd S t j } fdd| D }tj| I d H   j   d S )Nc                    s"   g | ]}t  |d  |d dqS )itemrf  )ZslackAlertingInstancerz  rf  )r   )r   rz  rU   r-   rD   r     s    z2SlackAlerting.async_send_batch.<locals>.<listcomp>)rs  r    valuesr2   gatherclear)r>   Zsquashed_queuetasksr-   rU   rD   async_send_batch  s   

zSlackAlerting.async_send_batchc                    s  ddl m } | }g }| j4 I dH  | j D ]\}}|d }| j|}|du r-q||d   }||jk r;q|d 	d}	|d 	d}
|d 	d}|d 	d}d	| d
|d  d| d|	 d| d|
 d|d  d|d  }t
dd}|dur|d| d7 }d|i}ddi}|d }t|tr|D ]}| j||||d qn| j||||d || q|D ]}| j|= qW d  I dH  dS 1 I dH sw   Y  dS )zFlush any digest buckets whose interval has expired.

        For each expired bucket, formats a digest summary message and
        appends it to the log_queue for delivery via the normal batching path.
        r   rd  Nr   rf   ri  rg  %Y-%m-%drk  z` (Digest)
Level: `r~   z
`
Start: ` z`
End: `z
`
Count: `rf  rl  rh  r$  rn  ry   r5  r-  r.  r6  ro  )re  rq  r;   r:   r6   r*   rQ   ra   Zdigest_intervalrr  r&  r'  r7   rN   rs  r  )r>   re  rq  Zflushed_keysr@   entryrx  ru  elapsedZstart_tsZend_ts
start_dateend_datery  rw  r7  r  r6  r/  r-   r-   rD   _flush_digest_buckets
  sj   


.z#SlackAlerting._flush_digest_bucketsc              
      sr   	 t | jI dH  z	|  I dH  W n ty0 } ztdt|  W Y d}~nd}~ww |  I dH  q)z:Override base periodic_flush to also flush digest buckets.TNzError flushing digest buckets: )	r2   sleepZflush_intervalr  re   r   debugrd   rt  )r>   rj   r-   r-   rD   rG   B  s    zSlackAlerting.periodic_flushc              
      s  zmd| j v rl|di pi }|di pi }|ddpd}|| }|}	t|tjrOt|drO|jdurOt|jdrO|jj}
|
durO|
d	krOt|	 |
 }	t|	t
rX|		 }	| t|d
|	tj dI dH  W dS W dS  ty } ztdt|  W Y d}~dS d}~ww )zLog deployment latencyr   rZ   r*  r   r\   usageNcompletion_tokensr   Fr   r   r   Z
updated_atzR[Non-Blocking Error] Slack Alerting: Got error in logging LLM deployment latency: )r'   rQ   r7   rb   ZModelResponsehasattrr  r  floatra   r   r   DeploymentMetricsr  get_utc_datetimere   r   rY  rd   )r>   r?   response_objrf   rg   rZ   r*  r  Z
response_sZfinal_valuer  rj   r-   r-   rD   async_log_success_eventL  sJ   




z%SlackAlerting.async_log_success_eventc           	   
      s  | di }| di pi }| dd}zid| jv rKz| t|ddtj dI dH  W n tyJ } zt	d	t
|  W Y d}~nd}~ww t| d
dtryd| jv re| j|d
 |dI dH  d| jv r|| j|d
 |dI dH  W dS W dS W dS  ty   Y dS w )z Log failure + deployment latencyrZ   r*  r   r\   r   TNr  zException raises -r  r  )r  r  r  )rQ   r'   r   r  rb   r  r  re   r   r  rd   r7   r   r  r  )	r>   r?   r  rf   rg   Z_litellm_paramsZ_model_infor  rj   r-   r-   rD   async_log_failure_eventr  sD   
 

z%SlackAlerting.async_log_failure_eventc                    s   d}| j jtjjddI dH }t }|du r'| j jtjj|dI dH  |S t|trN| j	j
}|| |krN| j|dI dH  | j jtjj|dI dH  d}|S )z[
        Returns:
        - True -> report sent
        - False -> report not sent
        FN)r@   r   r  )r   T)r$   r   r   Zreport_sent_keyr   r   r   r7   r  r1   r   r   )r>   rE   Zreport_sent_boolZreport_sentrv  Zinterval_secondsr-   r-   rD   _run_scheduler_helper  s.   
z#SlackAlerting._run_scheduler_helperc                    sh   |du s
| j du rdS d| j v r2	 | j|dI dH  t| jjd | jjd }t|I dH  qdS )z
        If 'daily_reports' enabled

        Ping redis cache every 5 minutes to check if we should send the report

        If yes -> call send_daily_report()
        Nr   T)rE   r   )r'   r  randomrandintr1   Zreport_check_intervalr2   r  )r>   rE   intervalr-   r-   rD   _run_scheduled_daily_report  s   


z)SlackAlerting._run_scheduled_daily_report7d
time_rangec              
      s>  | j du sd| jvrdS zddlm} t|dd }|d  dkr(tdtj 	 }|tj
|d }d	|d
 d|d
 }| jj|dI dH rRW dS ||d
|d
dI dH }|du sk|g g fkrnW dS |\}}	d|d d|d d| d}
|dur|
d7 }
|D ]}tt|d d}|
d|d  d| d7 }
q|	dur|
d7 }
|	D ]}tt|d d}|
d|d  d| d7 }
q| j|
dtji dI dH  | jj|dt|d I dH  W dS  ty } ztd!|  W Y d}~dS d}~w ty } ztd"|  W Y d}~dS d}~ww )#z
        Send a spend report for a configurable time range.

        Args:
            time_range: A string specifying the time range for the report, e.g., "1d", "7d", "30d"
        Nspend_reportsr    _get_spend_report_for_time_rangedz0Time range must be specified in days, e.g., '7d'daysZweekly_spend_report_sent_r  r  ro   r  r  u   *💸 Spend Report for `%m-%d-%Y - z` (z days)*

*Team Spend Report:*
total_spendr   Team: `r]  ` | Spend: `$r   
*Tag Spend Report:*
Tag: `individual_request_tagr|   r   r   r   zInvalid time range format: zError sending spend report: )r&   r'   7litellm.proxy.spend_tracking.spend_management_endpointsr  intlowerr>  re  rq  dater   rr  r$   r   ru   r  r   r   r  r   r   r   rY  re   )r>   r  r  r  todays_dater  _event_cache_key_respZspend_per_teamZspend_per_tag_spend_messager   _team_spend
_tag_spendverj   r-   r-   rD   send_weekly_spend_report  sf   
$z&SlackAlerting.send_weekly_spend_reportc              
      s  zddl m} ddlm} tj  }|jdd}||j|j	\}}|tj
|d d }d|d d	|d }| jj|d
I dH rIW dS ||d|ddI dH }|du sb|g g fkreW dS |\}	}
d|d d|d d}|	dur|d7 }|	D ]}|d }t|}t|d}|d|d  d| d7 }q|
dur|d7 }|
D ]}|d }t|}t|d}|d|d  d| d7 }q| j|dtji dI dH  | jj|ddt d d d I dH  W dS  ty } ztd!| W Y d}~dS d}~ww )"r  r   )
monthranger  r   )dayr  Zmonthly_spend_report_sent_r  r  ro   Nr  u    *💸 Monthly Spend Report for `r  r  z` *
r  r  r   r  r]  r  r   r  r  r  r|   r   r      <   r   $Error sending weekly spend report %s)calendarr  r  r  re  rq  r  replaceyearmonthr   rr  r$   r   r  ru   r   r   r  r   r   re   r   r  )r>   r  r  r  Zfirst_day_of_monthr  Zlast_day_of_monthr  r  Zmonthly_spend_per_teamZmonthly_spend_per_tagr  r   r  r  rj   r-   r-   rD   send_monthly_spend_report  sh   

z'SlackAlerting.send_monthly_spend_reportc              
      sx   z!ddl m} | I dH }d| }| j|dtji dI dH  W dS  ty; } ztd| W Y d}~dS d}~ww )z
        Helper to send fallback statistics from prometheus server -> to slack

        This runs once per day and sends an overview of all the fallback statistics
        r   )#get_fallback_metric_from_prometheusNz*Fallback Statistics:*
r|   r   r  )Z6litellm.integrations.prometheus_helpers.prometheus_apir  r   r   Zfallback_reportsre   r   rY  )r>   r  Z!falllback_success_info_prometheusZfallback_messagerj   r-   r-   rD   #send_fallback_stats_from_prometheusY  s&   

z1SlackAlerting.send_fallback_stats_from_prometheus	key_eventr^  c           
   
      s   zSd| d}|  }|d7 }| D ]\}}d|v r&|| d| d7 }q|d7 }|j}| D ]\}}|dkr;q2|| d| d7 }q2| j|d|i d	I d
H  W d
S  tym }	 ztd|	 W Y d
}	~	d
S d
}	~	ww )z
        Handles sending Virtual Key related alerts

        Example:
        - New Virtual Key Created
        - Internal User Updated
        - Team Created, Updated, Deleted
        ry   r   z*Action Done by:*
Z
created_byrm  z
*Arguments passed:*
Zuser_api_key_dictr   r   Nz-Error sending send_virtual_key_event_slack %s)r   r6   request_kwargsr   re   r   rY  )
r>   r  r   r^  r}   Zkey_event_dictr@   r   r  rj   r-   r-   rD   send_virtual_key_event_slacky  s:   z*SlackAlerting.send_virtual_key_event_slackc                    sv   |du rdS | dddkr9| dddkr9| dd}| jjd|d	d
I dH }|dur9|dks7|dkr9d	S dS )z[
        Returns True if the request is completed - either as a success or failure
        NFZlitellm_statusr\   successZfaillitellm_call_idzrequest_status:{}T)r@   
local_only)rQ   r$   r   r   )r>   r   r  statusr-   r-   rD   _request_is_completed  s   
z#SlackAlerting._request_is_completed)NNNNNNNrT   )NN)NNN)r  )G__name__
__module____qualname____doc__ZDEFAULT_ALERT_TYPESr   r   r  r   r   r   r
   rd   r8   r=   r"   rI   r  r  rP   rR   rV   rX   rY   rk   rx   r   r  r  r   boolr   r   r   r   r   r   r	   r   r   r   r   r   ZBaseOutageModelr  r   r  r  r   r+  r,  r   r   r?  ra  rc  r   r  r  rG   r  r  r  r  r  r  r  r   r  r  __classcell__r-   r-   rB   rD   r#   6   s   .

,


"8
. 

\
>
$

 
 
;"

h
M
 #8
&!$
ID 
/r#   ):r2   re  r&  r  r   r   typingr   r   r   r   r   r   r	   r
   Zopenair   rb   Zlitellm.litellm_core_utilsZ*litellm.litellm_core_utils.litellm_loggingZlitellm.typesZlitellm._loggingr   r   Zlitellm.caching.cachingr   Zlitellm.constantsr   Z(litellm.integrations.custom_batch_loggerr   Z5litellm.integrations.SlackAlerting.budget_alert_typesr   Z8litellm.integrations.SlackAlerting.hanging_request_checkr   Z*litellm.litellm_core_utils.duration_parserr   Z2litellm.litellm_core_utils.exception_mapping_utilsr   Z&litellm.llms.custom_httpx.http_handlerr   r   Zlitellm.proxy._typesr   r   r   r   r   Z)litellm.types.integrations.slack_alertingZemail_templates.templatesZbatching_handlerr   r    r  r!   Zlitellm.routerr"   Z_Routerr#   r-   r-   r-   rD   <module>   s>   (