o {q iok@sdZddlmZddlZddlZddlmZmZmZddl m Z m Z m Z m Z mZddlZddlZddlmZddlmZddlmZmZmZdd lmZmZmZmZmZdd lm Z e!e"Z#dddZ$GdddeeZ%dS)z#Base classes for OpenAI embeddings.) annotationsN)IterableMappingSequence)AnyLiteralOptionalUnioncast) Embeddings)run_in_executor)from_envget_pydantic_field_namessecret_from_env) BaseModel ConfigDictField SecretStrmodel_validator)Self num_textsinttokenslist[Union[list[int], str]]batched_embeddingslist[list[float]]indices list[int] skip_emptyboolreturnlist[Optional[list[float]]]c s"ddt|D}ddt|Dtt|D]%|r%t|dkr%q||||t|qg}t|D]J|}t|dkrV|dqDt|dkrd||dqDtfddt|D}tdd|Dd |fd d|DqD|S) NcSg|]}gqSr#.0_r#r#g/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/langchain_openai/embeddings/base.py z7_process_batched_chunked_embeddings..cSr"r#r#r$r#r#r'r(#r)rcs,g|]}tddt|DqS)css|] \}}||VqdSNr#)r%valweightr#r#r' AszA_process_batched_chunked_embeddings...)sumzip)r% embedding)inum_tokens_in_batch total_weightr#r'r(@s css|]}|dVqdS)Nr#r%r,r#r#r'r.Hz6_process_batched_chunked_embeddings..g?csg|]}|qSr#r#r6) magnituder#r'r(Is)rangelenappendr/r0) rrrrrresults embeddings_resultZaverager#)r2r8r3r4r'#_process_batched_chunked_embeddingss.      r?c@seZdZUdZedddZded<edddZded<dZd ed <dZ d ed < eZ d ed<ee dddddZ d ed< ede ddddZ d ed< ee ddddZd ed<ee ddddZd ed<dZded< ed ed!dddZd"ed#< ed$e d%d&gdddZd ed'< dZd(ed)<dZd*ed+<d,Zded-< d.Zded/< edd0d1Zd2ed3< dZded4<dZd5ed6< dZd ed7< d8Zd5ed9< eedZd:ed;< d8Zd5ed<< dZ d=ed><dZ!d?ed@<dAZ"dedB< dCZ#dedD< dZ$dEedF< dZ%dEedG< dZ&d5edH< e'dIddJdKZ(e)dLdMe*dodPdQZ+e)dRdMdpdTdUZ,e-dqdVdWZ.drd[d\Z/dd]dsdadbZ0dd]dsdcddZ1 dtdudedfZ2 dtdudgdhZ3dvdkdlZ4dvdmdnZ5dS)wOpenAIEmbeddingsu OpenAI embedding model integration. Setup: Install ``langchain_openai`` and set environment variable ``OPENAI_API_KEY``. .. code-block:: bash pip install -U langchain_openai export OPENAI_API_KEY="your-api-key" Key init args — embedding params: model: str Name of OpenAI model to use. dimensions: Optional[int] = None The number of dimensions the resulting output embeddings should have. Only supported in ``'text-embedding-3'`` and later models. Key init args — client params: api_key: Optional[SecretStr] = None OpenAI API key. organization: Optional[str] = None OpenAI organization ID. If not passed in will be read from env var ``OPENAI_ORG_ID``. max_retries: int = 2 Maximum number of retries to make when generating. request_timeout: Optional[Union[float, Tuple[float, float], Any]] = None Timeout for requests to OpenAI completion API See full list of supported init args and their descriptions in the params section. Instantiate: .. code-block:: python from langchain_openai import OpenAIEmbeddings embed = OpenAIEmbeddings( model="text-embedding-3-large" # With the `text-embedding-3` class # of models, you can specify the size # of the embeddings you want returned. # dimensions=1024 ) Embed single text: .. code-block:: python input_text = "The meaning of life is 42" vector = embeddings.embed_query("hello") print(vector[:3]) .. code-block:: python [-0.024603435769677162, -0.007543657906353474, 0.0039630369283258915] Embed multiple texts: .. code-block:: python vectors = embeddings.embed_documents(["hello", "goodbye"]) # Showing only the first 3 coordinates print(len(vectors)) print(vectors[0][:3]) .. code-block:: python 2 [-0.024603435769677162, -0.007543657906353474, 0.0039630369283258915] Async: .. code-block:: python await embed.aembed_query(input_text) print(vector[:3]) # multiple: # await embed.aembed_documents(input_texts) .. code-block:: python [-0.009100092574954033, 0.005071679595857859, -0.0029193938244134188] NT)defaultexcluderclient async_clientztext-embedding-ada-002strmodel Optional[int] dimensionsz Optional[str] deploymentZOPENAI_API_VERSION)rA api_version)default_factoryaliasopenai_api_versionbase_urlZOPENAI_API_BASE)rLrKopenai_api_baseZOPENAI_API_TYPE)rKopenai_api_typeZ OPENAI_PROXY openai_proxyirembedding_ctx_lengthapi_keyZOPENAI_API_KEYzOptional[SecretStr]openai_api_key organizationZ OPENAI_ORG_IDZOPENAI_ORGANIZATIONopenai_organizationz%Union[Literal['all'], set[str], None]allowed_specialz4Union[Literal['all'], set[str], Sequence[str], None]disallowed_speciali chunk_sizer5 max_retriestimeout)rArLz0Optional[Union[float, tuple[float, float], Any]]request_timeoutheadersrtiktoken_enabledtiktoken_model_nameFshow_progress_bardict[str, Any] model_kwargsrzUnion[Mapping[str, str], None]default_headersz!Union[Mapping[str, object], None] default_queryretry_min_secondsretry_max_secondszUnion[Any, None] http_clienthttp_async_clientcheck_embedding_ctx_lengthZforbidr#)extraZpopulate_by_nameZprotected_namespacesbefore)modevaluesr c Cst|}|di}t|D]*}||vrd|d}t|||vr8td|d|d|d||||<q||}|rLd|d }t|||d<|S) z>Build extra kwargs from additional params that were passed in.rbzFound z supplied twice.z WARNING! z/ is not default parameter. zJ was transferred to model_kwargs. Please confirm that z is what you intended.z Parameters za should be specified explicitly. Instead they were passed in as part of `model_kwargs` parameter.) rgetlist ValueErrorwarningswarnpop intersectionkeys)clsroZall_required_field_namesrl field_namemsgZinvalid_model_kwargsr#r#r' build_extras.    zOpenAIEmbeddings.build_extraafterrc Cs||jdvr d}t||jr|jnd|j|j|j|j|j|j d}|j rD|j s,|j rD|j }|j }|j }d|d|d|}t||j s|j ro|j sozddl}Wntyf}zd }t||d}~ww|j|j d |_ d |j i}tjd i||j|_ |js|j r|j szddl}Wnty}zd }t||d}~ww|j|j d |_ d |j i} tjd i|| j|_|S) z?Validate that api key and python package exists in environment.)ZazureZazure_adZazureadzEIf you are using Azure, please use the `AzureOpenAIEmbeddings` class.N)rSrUrNr[rZrcrdzwCannot specify 'openai_proxy' if one of 'http_client'/'http_async_client' is already specified. Received: openai_proxy=z http_client=z http_async_client=rzRCould not import httpx python package. Please install it with `pip install httpx`.)proxyrir#)rPrrrTZget_secret_valuerVrOr\rZrcrdrQrirjrChttpx ImportErrorZClientopenaiZOpenAIr=rDZ AsyncClientZ AsyncOpenAI) selfrzZ client_paramsrQrirjr~eZ sync_specificZasync_specificr#r#r'validate_environment!st          z%OpenAIEmbeddings.validate_environmentcCs(d|ji|j}|jdur|j|d<|S)NrFrH)rFrbrH)rparamsr#r#r'_invocation_params^s  z#OpenAIEmbeddings._invocation_paramstexts list[str]\} } |jd r| d d } |r|j | fi|}n|| }t dt ||j D]} ||| | |j || qq|jrzddlm}|t dt ||}Wntyt dt ||}Yn wt dt ||}|||fS)aTake the input `texts` and `chunk_size` and return 3 iterables as a tuple. We have `batches`, where batches are sets of individual texts we want responses from the openai api. The length of a single batch is `chunk_size` texts. Each individual text is also split into multiple texts based on the `embedding_ctx_length` parameter (based on number of tokens). This function returns a 3-tuple of the following: _iter: An iterable of the starting index in `tokens` for each *batch* tokens: A list of tokenized texts, where each text has already been split into sub-texts based on the `embedding_ctx_length` parameter. In the case of tiktoken, this is a list of token arrays. In the case of HuggingFace transformers, this is a list of strings. indices: An iterable of the same length as `tokens` that maps each token-array to the index of the original text in `texts`. r) AutoTokenizerzCould not import transformers python package. This is needed for OpenAIEmbeddings to work without `tiktoken`. Please install it with `pip install transformers`. )Zpretrained_model_name_or_pathF)Zadd_special_tokensZ cl100k_basecSsi|] \}}|dur||qSr+r#)r%kvr#r#r' s z.OpenAIEmbeddings._tokenize..)rWrXZ001  )tqdm)r_rFr^Z transformersrrrrZfrom_pretrained enumerateencoder9r:rRdecoder;tiktokenZencoding_for_modelKeyErrorZ get_encodingrWrXitemsendswithreplaceZencode_ordinaryr`Z tqdm.autor)rrrYrrZ model_namerrzZ tokenizerr2textZ tokenizedjZ token_chunkZ chunk_textencodingZencoder_kwargstokenr_iterr#r#r' _tokenizeesn              zOpenAIEmbeddings._tokenize)rYenginekwargsrc  s|pj}ij|||\}}}g} |D](} jjd d|| | |i} t| ts5| } | dd| dDqt t ||| |j } dd fdd fd d | DS)acGenerate length-safe embeddings for a list of texts. This method handles tokenization and embedding generation, respecting the set embedding context length and chunk size. It supports both tiktoken and HuggingFace tokenizer based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. inputcs|]}|dVqdSr1Nr#r%rr#r#r'r.r7z.dataNr list[float]csDdur jjdddi}t|ts|}|dddSNrrrr1r#)rCcreate isinstancedict model_dumpZaverage_embedded_cached_empty_embedding client_kwargsrr#r'empty_embeddings  zBOpenAIEmbeddings._get_len_safe_embeddings..empty_embeddingcsg|] }|dur |nqSr+r#r%rrr#r'r(sz=OpenAIEmbeddings._get_len_safe_embeddings..r#r r) rYrrrCrrrrextendr?r:r rrrrYr _chunk_sizerrrrr2responser=r#rrrrr'_get_len_safe_embeddingss&    z)OpenAIEmbeddings._get_len_safe_embeddingsc s|pj}ij|tdj||IdH\}}}g} tdt||D]+} jjd d|| | |iIdH} t| t sD| } | dd| dDq%t t||| |j } ddfd d fd d | DIdHS)aAsynchronously generate length-safe embeddings for a list of texts. This method handles tokenization and asynchronous embedding generation, respecting the set embedding context length and chunk size. It supports both `tiktoken` and HuggingFace `tokenizer` based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. Nrrcsrrr#rr#r#r'r.r7z=OpenAIEmbeddings._aget_len_safe_embeddings..rr rcsLdur$jjdddiIdH}t|ts|}|dddSr)rDrrrrrrr#r'r%s  zCOpenAIEmbeddings._aget_len_safe_embeddings..empty_embeddingcs&g|]}|dur |nIdHqSr+r#rrr#r'r(0s$z>OpenAIEmbeddings._aget_len_safe_embeddings..r#r)rYrr rr9r:rDrrrrrr?rrr#rr'_aget_len_safe_embeddingss,     z*OpenAIEmbeddings._aget_len_safe_embeddingsc Ks|p|j}i|j|}|jsDg}tdt||D](}|jjdd||||i|}t|ts5| }| dd|dDq|St t |j } |j|f| |d|S) aCall out to OpenAI's embedding endpoint for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. kwargs: Additional keyword arguments to pass to the embedding API. Returns: List of embeddings, one for each text. rrcsrrr#rr#r#r'r.Jr7z3OpenAIEmbeddings.embed_documents..rrrYNr#)rYrrkr9r:rCrrrrrr rErIr rrrYrZ chunk_size_rr=r2rrr#r#r'embed_documents2s,    z OpenAIEmbeddings.embed_documentsc s|p|j}i|j|}|jsHg}tdt||D]+}|jjdd||||i|IdH}t|ts9| }| dd|dDq|St t |j } |j|f| |d|IdHS) aCall out to OpenAI's embedding endpoint async for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. kwargs: Additional keyword arguments to pass to the embedding API. Returns: List of embeddings, one for each text. rrNcsrrr#rr#r#r'r.lr7z4OpenAIEmbeddings.aembed_documents..rrr#)rYrrkr9r:rDrrrrrr rErIrrr#r#r'aembed_documentsTs.     z!OpenAIEmbeddings.aembed_documentsrrcKs|j|gfi|dS)aCall out to OpenAI's embedding endpoint for embedding query text. Args: text: The text to embed. kwargs: Additional keyword arguments to pass to the embedding API. Returns: Embedding for the text. r)r)rrrr#r#r' embed_queryvs zOpenAIEmbeddings.embed_querycs$|j|gfi|IdH}|dS)a Call out to OpenAI's embedding endpoint async for embedding query text. Args: text: The text to embed. kwargs: Additional keyword arguments to pass to the embedding API. Returns: Embedding for the text. Nr)r)rrrr=r#r#r' aembed_querys zOpenAIEmbeddings.aembed_query)rorar r)r r)r ra)rrrYrr r) rrrrErYrGrrr rr+)rrrYrGrrr r)rrErrr r)6__name__ __module__ __qualname____doc__rrC__annotations__rDrFrHrIr rMrOrPrQrRrrTrVrWrXrYrZr\r]r^r_r`rrbrrcrdrfrhrirjrkrZ model_configr classmethodr{rpropertyrrrrrrrrr#r#r#r'r@Ns R                          <  g98 # " r@) rrrrrrrrrrr r!)&r __future__rloggingrscollections.abcrrrtypingrrrr r rrZlangchain_core.embeddingsr Zlangchain_core.runnables.configr Zlangchain_core.utilsr rrZpydanticrrrrrZtyping_extensionsr getLoggerrloggerr?r@r#r#r#r's       9