o `+ i#)@sdZddlZddlmZddlmZmZmZddlm Z ddl m Z m Z m Z mZmZddlmZmZddlmZdd lmZmZdd lmZdd lmZmZdd lmZmZdd l m!Z!ddl"m#Z#edZ$ededZ%dee$de e$ge%fdee$fddZ&GdddeZ'dS)zo Ensemble retriever that ensemble the results of multiple retrievers by using weighted Reciprocal Rank Fusion N) defaultdict)HashableIterableIterator)chain)AnyCallableOptionalTypeVarcast)#AsyncCallbackManagerForRetrieverRunCallbackManagerForRetrieverRun)Document) BaseRetriever RetrieverLike)RunnableConfig) ensure_config patch_config)ConfigurableFieldSpecget_unique_config_specs)model_validator)overrideTH)bounditerablekeyreturnccs6t}|D]}||}|vr|||VqdS)a Yield unique elements of an iterable based on a key function. Args: iterable: The iterable to filter. key: A function that returns a hashable key for each element. Yields: Unique elements of the iterable based on the key function. N)setadd)rrseenekr#i/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/langchain/retrievers/ensemble.py unique_by_key%s  r%c @szeZdZUdZeeed<eeed<dZe ed<dZ e e ed<e deefd d Zed d ed ee efdefddZe d%de de ededeefddZe d%de de ededeefddZde dedeefddZde dedeefddZddde dede edeefddZddde dede edeefd d!Zd"eeedeefd#d$Z dS)&EnsembleRetrieveraeRetriever that ensembles the multiple retrievers. It uses a rank fusion. Args: retrievers: A list of retrievers to ensemble. weights: A list of weights corresponding to the retrievers. Defaults to equal weighting for all retrievers. c: A constant added to the rank, controlling the balance between the importance of high-ranked items and the consideration given to lower-ranked items. Default is 60. id_key: The key in the document's metadata used to determine unique documents. If not specified, page_content is used. retrieversweights<cNid_keyrcCstdd|jDS)z+List configurable fields for this runnable.css |] }|jD]}|VqqdSN) config_specs).0 retrieverspecr#r#r$ Ns  z1EnsembleRetriever.config_specs..)rr'selfr#r#r$r-KszEnsembleRetriever.config_specsbefore)modevaluescCs,|dst|d}d|g||d<|S)Nr(r')getlen)clsr6Z n_retrieversr#r#r$ set_weightsRs  zEnsembleRetriever.set_weightsinputconfigkwargsc Ksddlm}t|}|j|dd|dd|dg|j|di|jd}|jd|fd |d p5|i|}z |j |||d }Wnt yV}z| |d}~ww|j |fi||S) Nr)CallbackManager callbacksverboseFtagsmetadatarAZinheritable_tagsZ local_tagsZinheritable_metadataZlocal_metadatanamerun_name run_managerr=) langchain_core.callbacksr?r configurer8rBrCon_retriever_startget_name rank_fusion Exceptionon_retriever_erroron_retriever_end) r3r<r=r>r?callback_managerrHresultr!r#r#r$invokeZs@      zEnsembleRetriever.invokec sddlm}t|}|j|dd|dd|dg|j|di|jd}|jd|fd |d p6|i|IdH}z |j |||d IdH}Wnt y`}z | |IdHd}~ww|j |fi|IdH|S) Nr)AsyncCallbackManagerr@rAFrBrCrDrErFrG) rIrTrrJr8rBrCrKrL arank_fusionrNrOrP) r3r<r=r>rTrQrHrRr!r#r#r$ainvokesJ       zEnsembleRetriever.ainvokequeryrHcCs |||S)z Get the relevant documents for a given query. Args: query: The query to search for. Returns: A list of reranked documents. )rMr3rWrHr#r#r$_get_relevant_documentss z)EnsembleRetriever._get_relevant_documentscs|||IdHS)z Asynchronously get the relevant documents for a given query. Args: query: The query to search for. Returns: A list of reranked documents. N)rUrXr#r#r$_aget_relevant_documentssz*EnsembleRetriever._aget_relevant_documents)r=csNfddt|jD}tt|D] }dd||D||<q||S)z Retrieve the results of the retrievers and use rank_fusion_func to get the final result. Args: query: The query to search for. Returns: A list of reranked documents. c 6g|]\}}|tjd|dddqSZ retriever_r7)tag)r@)rSr get_childr.ir/r=rWrHr#r$ z1EnsembleRetriever.rank_fusion..cSs*g|]}t|trttt|dn|qS) page_content) isinstancestrrr r.docr#r#r$rbs) enumerater'ranger9weighted_reciprocal_rankr3rWrHr=Zretriever_docsr`r#rar$rMs  zEnsembleRetriever.rank_fusioncs\tjfddt|jDIdH}tt|D] }dd||D||<q||S)z Asynchronously retrieve the results of the retrievers and use rank_fusion_func to get the final result. Args: query: The query to search for. Returns: A list of reranked documents. c r[r\)rVrr^r_rar#r$rb rcz2EnsembleRetriever.arank_fusion..NcSs$g|]}t|tst|dn|qSrd)rfrrhr#r#r$rbs)asynciogatherrjr'rkr9rlrmr#rar$rUs  zEnsembleRetriever.arank_fusion doc_listscst|tjkrd}t|ttt|jD](\}}t|ddD]\}}jdur0|jn|j j||j 7<q#qt |}t t|fdddfddd S) a Perform weighted Reciprocal Rank Fusion on multiple rank lists. You can find more details about RRF here: https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf Args: doc_lists: A list of rank lists, where each rank list contains unique items. Returns: list: The final aggregated list of items sorted by their weighted RRF scores in descending order. zFs  z.Tcs"jdur |jS|jjSr,rrrsZ rrf_scorer3r#r$rtMs  )reverser)r9r( ValueErrorrfloatziprjr+rerCr*r from_iterablesortedr%)r3rpmsgZdoc_listweightZrankriZall_docsr#rur$rl s.      z*EnsembleRetriever.weighted_reciprocal_rankr,)!__name__ __module__ __qualname____doc__listr__annotations__rxr*intr+r rgpropertyrr-r classmethoddictrr;rrrrSrVr rYr rZrMrUrlr#r#r#r$r&6s     $ (   - * r&)(rrn collectionsrcollections.abcrrr itertoolsrtypingrrr r r rIr r Zlangchain_core.documentsrZlangchain_core.retrieversrrZlangchain_core.runnablesrZlangchain_core.runnables.configrrZlangchain_core.runnables.utilsrrZpydanticrZtyping_extensionsrrrr%r&r#r#r#r$s$       (