o ưi-@sddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z ddlZddlZddlmZddlmZddlmZe rMddlmZne ZejedZedd Zed krfedd lmZdZ Gd d d eZ!d e e"de#de"fddZ$GdddZ%e&ej'dddddej'dddddej'dedddej'd dd!d"ej'd#d$d%d"ej'd&d'dd(d"ej'd)dd*d"ej'd+dd(d"ej'd,dd-d"ej'd.d/e(d0d1ej'd2d3d/e(d4d5d6ej'd7d3d/e(d8d9d6ej'd:d3d/e(d;d<ej'd=de)d>d?ej'd@de*dAd?ej'dBde*dCd?ej'dDd/dEdFej'dGd/dHdFej'dIdJddKd"ej'dLde)dMd?ej'dNd/e(dOd?ej'dPde"dQd?ej'dRdSd3d/e(dTd<ej'dUd/dVdWej'dXd/dYdWej'dZd3d/d[d\ej'd]d3d/d^d\ej'd_d`e*dad?ej'dbd3d/dcd\ej'ddd3d/ded\ej'dfde"dgdhdiej'djde"dkdldiej'dmde"dnd?ej'dod/d3dpdqej'drd/d3dsdqej'dtd/d3dudqej'dvde*dwdxdiej'dyde*dzd{did|e(fd}d~Z+e,dkre+dSdS)N) TYPE_CHECKINGAnyOptionalUnion) load_dotenv)!DEFAULT_NUM_WORKERS_LITELLM_PROXY)get_secret_bool)FastAPIzlitellm.secretsZ LITELLM_MODEZDEV)Enumc@seZdZdZdZdS)LiteLLMDatabaseConnectionPool <N)__name__ __module__ __qualname__database_connection_pool_limit database_connection_pool_timeoutrrN/home/app/Keep/.python/lib/python3.10/site-packages/litellm/proxy/proxy_cli.pyr #sr urlparamsreturncCsddlm}|d||d|t|tr|dkr&|ddSt|}t|j}| |tj |dd}t |j |d }|S) Nr)verbose_proxy_loggerzurl: zparams: zLappend_query_params received empty or non-string URL, returning empty stringT)doseq)query) litellm._loggingrdebug isinstancestrwarningurlparseparse_qsrupdate urlencode urlunparse_replace)rrr parsed_urlZ parsed_queryZ encoded_query modified_urlrrrappend_query_params(s     r)c@seZdZeddZeddZededededee effd d Z e d$deded e ed e ede f ddZ e d%dededededede ef ddZe d%dedededededede efddZeddZeddZedd Zeded!e e dd fd"d#Zd S)&ProxyInitializationHelperscCs"tjd}td|ddS)Nlitellmz LiteLLM: Current Version =  ) importlibmetadataversionclickecho)Z pkg_versionrrr_echo_litellm_version;s z0ProxyInitializationHelpers._echo_litellm_versioncCs<tdtjd|d|dd}ttj|dddS)Nz) LiteLLM: Health Testing models in confighttp://:z/health)r)indent)printhttpxgetjsondumps)hostportresponserrr_run_health_check@sz,ProxyInitializationHelpers._run_health_checkr<r=modeltestc Cs|pd}td|ddl}d|d|}t|tr |}ntd|jd|d}|jjj |d d d gd d }td|t d||jjj |d d d gdd} | D] } td| qXt d|jj |d d} t | dS)Nz gpt-3.5-turbozE LiteLLM: Making a test ChatCompletions request to your proxy. Model=rr3r4zInvalid test valuez My API Key)Zapi_keybase_urluserz*this is a test request, write a short poem)Zrolecontent)r@messages max_tokensz LiteLLM: response from proxy zN LiteLLM: Making a test ChatCompletions + streaming r equest to proxy. Model=T)r@rFstreamz'LiteLLM: streaming response from proxy z$ making completion request to proxy)r@prompt) r0r1openairr ValueErrorZOpenAIZchatZ completionscreater7) r<r=r@rAZ request_modelrJapi_baseclientr>Zstream_responsechunkZcompletion_responserrr_run_test_chat_completionFsJ    z4ProxyInitializationHelpers._run_test_chat_completionN log_configkeepalive_timeoutrcCsdddl}ddlm}d||d}|dur td|||d<n|jr(||d<|dur0||d<|S) z8 Get the arguments for `uvicorn` worker rN)_get_uvicorn_json_log_configzlitellm.proxy.proxy_server:app)appr<r=zUsing log_config: rQZtimeout_keep_alive)r+rrSr7 json_logs)r<r=rQrRr+rS uvicorn_argsrrr_get_default_unvicorn_init_args|s   z:ProxyInitializationHelpers._get_default_unvicorn_init_argsrTssl_certfile_pathssl_keyfile_pathciphersc Csddl}ddlm}ddlm}td|d|d|} |d|g| _|durG|durGtd|d |d || _|| _|durG|| _ | ||| dS) z5 Initialize litellm with `hypercorn` rN)serve)Config)LiteLLM Proxy: Starting server on r4z using Hypercorn /LiteLLM Proxy: Using SSL with certfile:  and keyfile:  ) asyncioZhypercorn.asyncior[Zhypercorn.configr\r7bindcertfilekeyfilerZrun) rTr<r=rXrYrZrar[r\configrrr_init_hypercorn_servers"  z1ProxyInitializationHelpers._init_hypercorn_server num_workersmax_requests_before_restartc stjdkrnddl}Gfddd|jjj}td|dd|d |d|d d d d dd} |dur<|| d<tjdrRddl m fdd} | | d<|durm|durmtd|d|d|| d<|| d<||| d dS)z- Run litellm with `gunicorn` ntrNcs0eZdZdfdd ZddZddZZS) zNProxyInitializationHelpers._run_gunicorn_server..StandaloneApplicationNcsh|pi|_||_tdd}|d}ttdtd|dtdtdddS) Nz curl --location 'http://0.0.0.0:z/chat/completions' \ai --header 'Content-Type: application/json' \ --data ' { "model": "gpt-3.5-turbo", "messages": [ { "role": "user", "content": "what llm are you" } ] }' zLiteLLM: Test your local proxy with: "litellm --test" This runs an openai.ChatCompletion request to your proxy [In a new terminal tab] z8LiteLLM: Curl Command Test for your local proxy z  z;Docs: https://docs.litellm.ai/docs/simple_proxy z5See all Router/Swagger docs on http://0.0.0.0:)options applicationsuper__init__r7)selfrTrkZ _endpoint_strZ curl_command) __class__r=rrrns,     zWProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.__init__csZjdurfddjD}ni}|D]\}}jdur*j||qdS)Ncs*i|]\}}|jjvr|dur||qSN)cfgsettings).0keyvaluerorr s znProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.load_config..)rrrkitemssetlower)rorfrurvrrwr load_configs   zZProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.load_configcSs|jSrq)rlrwrrrloadszSProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.loadrq)rrrrnr|r} __classcell__r)r=)rprStandaloneApplications&rr]r4z with z workers zuvicorn.workers.UvicornWorkerT-iXz+%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s)rbworkersZ worker_classZpreloadZ accesslogtimeoutZaccess_log_formatZ max_requestsPROMETHEUS_MULTIPROC_DIRmark_worker_exitcs|jdSrq)pid)serverZworkerrrr child_exitszCProxyInitializationHelpers._run_gunicorn_server..child_exitrr^r_r`rcrd)rTrk) osnameZgunicorn.app.baserTbaseZBaseApplicationr7environr9 litellm.proxy.prometheus_cleanuprre) r<r=rTrhrXrYriZgunicornrZgunicorn_optionsrr)rr=r_run_gunicorn_servers8 9     z/ProxyInitializationHelpers._run_gunicorn_serverc Csz'ddg}ttjd}tj|||dWdWdS1s wYWdStyB}ztd|dWYd}~dSd}~ww)Nollamar[w)stdoutstderrzh LiteLLM Warning: proxy started with `ollama` model `ollama serve` failed with Exceptionz-. Ensure you run `ollama serve` )openrdevnull subprocessPopen Exceptionr7)commandrerrr_run_ollama_serve*s&z,ProxyInitializationHelpers._run_ollama_servecCsNddl}||j|j}|d|fdkWdS1s wYdS)Nr localhost)socketAF_INET SOCK_STREAM connect_ex)r=rsrrr_is_port_in_use8s$z*ProxyInitializationHelpers._is_port_in_usecCstjdvrdSdS)zBHelper function to determine the event loop type based on platform)win32cygwincliNZuvloop)sysplatformrrrr_get_loop_type?s z)ProxyInitializationHelpers._get_loop_typelitellm_settingsc Csddl}|dks |durdS|dpg}|dpg}|dp"g}|||}d|vr/dSddlm}tjd p@tjd }| } |sTtj|d }|tjd <tj |d d ||| rcdnd} t d| d|dS)z Auto-create PROMETHEUS_MULTIPROC_DIR when running with multiple workers and prometheus is configured as a callback. rN callbacksZsuccess_callbackZfailure_callbackZ prometheus)wipe_directoryrZprometheus_multiproc_dirZlitellm_prometheus_multiprocT)exist_okz Auto-createdzUsing existingz LiteLLM: z PROMETHEUS_MULTIPROC_DIR=) tempfiler9rrrrpathjoin gettempdirmakedirsr7) rhrrrZsuccess_callbacksZfailure_callbacksZ all_callbacksrZ multiproc_dirZ auto_createdactionrrr%_maybe_setup_prometheus_multiproc_dirFs.       z@ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir)NNrq)rrr staticmethodr2r?rintrboolrPrdictrWr rgrrrrrrrrrr*:s    5  " n   r*z--hostz0.0.0.0z!Host for the server to listen on.ZHOST)defaulthelpenvvarz--portzPort to bind the server to.ZPORTz --num_workerszfNumber of uvicorn / gunicorn workers to spin up. Default is 1 (from DEFAULT_NUM_WORKERS_LITELLM_PROXY)Z NUM_WORKERSz --api_basez API base URL.)rrz --api_versionz2024-07-01-previewz$For azure - pass in the api version.z--modelz-mz)The model name to pass to litellm expectsz--aliaszThe alias for the model - use this to give a litellm model name (e.g. "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama")z --add_keyz --headerszheaders for the API callz--saveTzSave the model-specific config)is_flagtyperz--debugFzTo debug the inputDEBUG)rrrrrz--detailed_debugzTo view detailed debug logsZDETAILED_DEBUGz --use_queuez)To use celery workers for async endpoints)rrrrz --temperaturezSet temperature for the model)rrrz --max_tokenszSet max tokens for the modelz--request_timeoutz+Set timeout in seconds for completion callsz --drop_paramszDrop any unmapped params)rrz--add_function_to_promptz5If function passed but unsupported, pass it as promptz--configz-cz]Path to the proxy configuration file (e.g. config.yaml). Usage `litellm --config config.yaml`z --max_budgetz`Set max budget for API calls - works for hosted models like OpenAI, TogetherAI, Anthropic, etc.`z --telemetryzZHelps us know if people are using this feature. Turn this off by doing `--telemetry False`z --log_configz&Path to the logging configuration filez --versionz-vzPrint LiteLLM versionz--healthz:Make a chat/completions request to all llms in config.yaml) flag_valuerz--testz4proxy chat completions url to make a test request toz --test_asyncz9Calls async endpoints /queue/requests and /queue/response)rrrz--iam_token_db_authz!Connects to RDS DB with IAM tokenz--num_requestsr z-Number of requests to hit async endpoint withz--run_gunicornzTStarts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)z--run_hypercornz@Starts proxy via hypercorn, instead of uvicorn (supports HTTP/2)z--ssl_keyfile_pathz^Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxyZSSL_KEYFILE_PATH)rrrrz--ssl_certfile_pathz_Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxyZSSL_CERTFILE_PATHz --ciphersz!Ciphers to use for the SSL setup.z--use_prisma_db_pushzHUse prisma db push instead of prisma migrate for database schema updates)rrrz--localzfor local debuggingz--skip_server_startupzASkip starting the server after setup (useful for migrations only)z--keepalive_timeoutzSSet the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter)ZKEEPALIVE_TIMEOUTz--max_requests_before_restartz]Restart worker after this many requests (uvicorn: limit_max_requests, gunicorn: max_requests)ZMAX_REQUESTS_BEFORE_RESTARTuse_prisma_db_pushc&P Cst}&|rddlm}'m}(m})m}*nHzddlm}'m}(m})m}*Wn9ty4}+ztd|+dd}+~+wtyY}+zdt|+vrC|+ddlm}'m}(m})m}*WYd}+~+nd}+~+ww|durdt dS|rrd|vrr|durrt |dur~t ||dS|durt ||||dS|rt|}|*d[id |d |d |d |d | d| d| d| d| d|d|d|d|d|d|d|d|zddl},Wn tytdwd}-d}.i}/|stdr5ddlm}0td}1td }2td!}3td"}4td#}5|0|1|2|3d$}6d%|3d&|6d'|1d&|2d(|4 }7|5r+|7d)|57}7|7tjd*<d+tjd<dd,lm}8td-ddur^td-d+kr^|8}9|9D] \}:};|;tj|:<qS|durJ zddl}|>d0d}?|?durd1|?vr|?d1durddl}@d|@_ |@!|>d2i}/|/duri}/|/d3d}A|Adurddl}@|'d[i|A|@_"|/r|/d4d}B|=j#|B|d5|/d6d}C|Cdurtd*durdd7l$m%}D|D}C|Cr|Ctjd*<|/d8t&j'j(}-|/d9t&j)j(}.|Cr:|C*d:r:t+}Et,j-.dtj-/d;ddl}@ddddurzMdd?l3m4}Gtd*ddur|-|.d@}H|Gd*dd=}Ct5|C|H}I|Itjd*<td>ddur|-|.d@}Htd>}Ct5|C|H}I|Itjd><t6jdAgddBd}JWn t7ydC}JYnw|Jr ddDl8m9}KddEl:m;}Lm<}M|M|/dFdCur|KddGn |Lj=|" dHnt>dI|dJkrt ?|rt@AdKdL}ddl}@| dur,|@BddMlCm})t jD||r:|?nddN|#rGt>dOdSt jE|||!|$dP}N|%durY|%|NdQ<|dCur|dCur|dur|durt>dR|dS|dT||NdU<||NdV<t F}O|Or|O|NdW<|,jd[i|NdX|idS|durt jG|||)||||%dYdS|durt jH|)||||| dZdSdS)\Nr)KeyManagementSettings ProxyConfigrTsave_worker_configrzMissing dependency z$. Run `pip install 'litellm[proxy]'`zlitellm[proxy]Trr@aliasrM api_versionrdetailed_debug temperaturerGrequest_timeout max_budget telemetry drop_paramsadd_function_to_promptheaderssaverf use_queuezLuvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`dr ZIAM_TOKEN_DB_AUTH)generate_iam_auth_tokenZ DATABASE_HOSTZ DATABASE_PORTZ DATABASE_USERZ DATABASE_NAMEZDATABASE_SCHEMA)db_hostdb_portdb_userz postgresql://r4@/z?schema=Z DATABASE_URLTrue)decrypt_env_varZ USE_AWS_KMSz?yaml needs to be imported. Run - `pip install 'litellm[proxy]'`)config_file_pathrrUgeneral_settingskey_management_settingskey_management_system)rr database_url)$construct_database_url_from_env_varsrrz os.environ/z../..)get_secret_str) default_valueZ DIRECT_URL) get_secret)Zconnection_limit pool_timeoutZprisma)capture_outputF)check_prisma_schema_diff) PrismaManagershould_update_prisma_schemaZdisable_prisma_schema_update)Zdb_url)Z use_migratezYUnable to connect to DB. DATABASE_URL found in environment, but prisma package not found.rii)rT)rhrz>LiteLLM: Setup complete. Skipping server startup as requested.)r<r=rQrRZlimit_max_requestsr^r_r`Z ssl_keyfileZ ssl_certfileloopr)r<r=rTrhrXrYri)rTr<r=rXrYrZr)IlocalsZ proxy_serverrrrTrModuleNotFoundError ImportErrorrr*r2rr?rPr:loadsuvicornrrZ litellm.proxy.auth.rds_iam_tokenrrgetenvrZ*litellm.secret_managers.aws_secret_managerrryrareZ get_configr9r+rUZ _turn_on_jsonZ_key_management_settingsZinitialize_secret_managerZlitellm.proxy.utilsrr rrvr startswithgetcwdrrinsertabspathrchdirrlitellm.secret_managers.mainrr)rFileNotFoundErrorZ litellm.proxy.db.check_migrationrZlitellm.proxy.db.prisma_clientrrZsetup_databaser7rrandomrandintZ_turn_on_debugZlitellm.proxy.proxy_serverrrWrrrg)Pr<r=rMrr@rZadd_keyrrrrrrGrrrrfrrrAlocalrhZ test_asyncZiam_token_db_auth num_requestsrZhealthr/Z run_gunicornZ run_hypercornrYrXrZrQrZskip_server_startuprRriargsrrrTrrrZdb_connection_pool_limitZdb_connection_timeoutrrrrrZdb_nameZ db_schematokenZ_db_urlrZ new_env_varkvra proxy_config_configrr+rrrrZ original_dirrrrr(Zis_prisma_runnablerrrrVZ loop_typerrr run_serverps g  $                "                                        r__main__)-r-r:rrrr urllib.parseparser!typingrrrrr0r8ZdotenvrZlitellm.constantsrrrZfastapir rappendrZconfig_filenamerZ litellm_modeenumr rr rrr)r*roptionrfloatrrrrrrrs      8#X |