o )i @sddlmZmZddlmZddlmZmZmZm Z m Z m Z ddl m Z mZddlmZddlmZddlmZmZmZmZmZmZmZmZddlZddlZdd lm Z dd l!m"Z"dd l#m$Z$dd l%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7ddl8m9Z9m:Z:m;Z;mZ>ddl?m@Z@ddlAmBZBddlCmDZDe$eEZFedeGeHeIZJeeGeHeIfZK e GdddZLGdddZMeeKeLfZN e GdddeeJZOeeKeOfZP eeeIgePfePfZQ GdddeGeZRe Gd d!d!eZSe Gd"d#d#eSZTe Gd$d%d%eSZUed&d'dd(d)e&d*eGd+eeVd,eHeIfd-d.ZWed&d'dd/d)e&d0eXeId1fd2eeVd,eGfd3d4ZYGd5d6d6eZZGd7d8d8eZ[ed9eeZe[fd:Z\d;e e\d,eeGeHe\ffdd?d?Z^e Gd@dAdAZ_e GdBdCdCZ`GdDdEdEeZad0eHeIdFeHeId,eeafdGdHZbd0eHeIdFeHeIdIeHeId,eHeIfdJdKZce dLdMGdNdOdOeZde dLdMGdPdQdQedZee dLdMGdRdSdSedZfe dLdMGdTdUdUedZge GdVdWdWZhdXeHeIdYe e`d,e edfdZd[ZidXeGdYe e`d,e edfd\d]ZjdXeKd^e eGe edfd,eHedfd_d`ZkdXeJd^e eGe edfdae eGeIfd,eHeJfdbdcZldXeHeId^e eGe edfdae eGeIfd,eHeIfdddeZmdXeGd^e eGe edfdae eGeIfd,eGfdfdgZndhe eGe e`fdXeHeIdae eGeIfd,e ehfdidjZodhe eGe e`fdXeHeIdae eGeIfd,e eGeHehffdkdlZpGdmdndne-Zqee6eGfZrGdodpdpZsedqesd:ZteueGeHeGfZv GdrdsdseeetZwGdtduduewetZxdS)v)ABCabstractmethod) defaultdict)Callable Generator ItemsViewIterableMappingSequence) dataclassfield)Enum) lru_cache) TYPE_CHECKINGGeneric NamedTupleOptionalProtocolTypeVarUnioncastN) assert_never)InputProcessingContext) init_logger) AnyTokenizer decode_tokens encode_tokens)flatten_2d_lists full_groupby)MultiModalCache)MultiModalHasher)MultiModalDataDictMultiModalEncDecInputsMultiModalFieldConfigMultiModalInputsMultiModalKwargsMultiModalKwargsItemPlaceholderRange)DictEmbeddingItemsEmbeddingItemsMultiModalDataItemsMultiModalDataParser)PretrainedConfig) BatchFeature)ProcessorMixin)BaseDummyInputsBuilder_Sc@s*eZdZUdZeeegeefe d<dS) PromptIndexz#Resolves to an index in the prompt.get_match_indexN) __name__ __module__ __qualname____doc__rr PromptSeqrint__annotations__r;r;f/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/multimodal/processing.pyr2.s r2c@sFeZdZedefddZededefddZedefddZd S) PromptIndexTargetsreturncC tddS)z Resolves to the start of the prompt (before the first token). This results in a match even if the prompt is empty. cSdSNrr;tokpromptr;r;r<=z*PromptIndexTargets.start..r2r;r;r;r<start6 zPromptIndexTargets.startseqcs&dtdtdttffdd }t|S)zN Resolves to a location in the prompt after the given prefix. tokenizerrDr>cs\}t|trt|tst||}n t|trt||dd}t|}|d||kr,|SdSNFadd_special_tokens) isinstancestrrrlen)rKrDprefix match_idxrJr;r<r3Es    z2PromptIndexTargets.prefix..get_match_index)rr8rr9r2)rJr3r;rTr<rR?szPromptIndexTargets.prefixcCr?)z Resolves to the end of the prompt (after the last token). This results in a match even if the prompt is empty. cSst|SNrQrBr;r;r<rEbsz(PromptIndexTargets.end..rGr;r;r;r<end[rIzPromptIndexTargets.endN) r4r5r6 staticmethodr2rHr8rRrWr;r;r;r<r=4sr=c@seZdZUdZeed< dZeedge j fed< e deddfd d Z e ded e ddfd d Ze dededdfddZdS)PromptUpdateDetailszEDetails about the token sequence or text that are part of the update.fullN_BoundPromptSequenceis_embedrJr>zPromptUpdateDetails[_S]cCs t|dS)N)rZrYrTr;r;r<from_seq~ zPromptUpdateDetails.from_seq embed_textcs$dddtjffdd }t||dS)NrZr[r>cs&t|j}tt|jt|SrU)rrKtorchisintensor token_ids)rZZembed_token_idsr`r;r<r\s  z1PromptUpdateDetails.select_text..is_embedrZr\)raTensorrY)rJr`r\r;rer< select_texts zPromptUpdateDetails.select_textembed_token_idcst|fdddS)Ncst|jkSrU)rarcrd)frir;r<rEsz5PromptUpdateDetails.select_token_id..rfr])rJrir;rkr<select_token_ids z#PromptUpdateDetails.select_token_id)r4r5r6r7r1r:r\rrrargrXr^rPrhr9rlr;r;r;r<rYks0   rYc@seZdZdZdZdS) UpdateModeinsertreplaceN)r4r5r6INSERTREPLACEr;r;r;r<rmsrmc@sdeZdZUdZeed< eed< eede fddZ eede fddZ d e dd fd d Zd S) PromptUpdatezA Defines how to update a prompt with placeholder tokens. modalitytargetr>cCtz3The placeholder tokens that are part of the update.NotImplementedErrorselfr;r;r<contentzPromptUpdate.contentcCruz!Defines how to update the prompt.rwryr;r;r<moder|zPromptUpdate.moderKBoundPromptUpdatecCs t||dS)N)_originrKr)rzrKr;r;r<bindszPromptUpdate.bindN)r4r5r6r7rPr: PromptTargetpropertyrPromptUpdateContentr{rmr~rrr;r;r;r<rrs rrc@JeZdZUdZeddZeed< edefddZ ede fdd Z d S) PromptInsertiona Defines how to insert placeholder tokens into a prompt. Example: For each image, insert a number of ```` feature placeholders equal to the feature size of the vision encoder after the ```` token: ```python PromptInsertion( modality="image", target="", insertion="" * image_feature_size, ) ``` Insert these tokens at the start of the prompt: ```python PromptInsertion( modality="image", target=PromptIndexTargets.start(), insertion="" * image_feature_size, ) ``` Insert these tokens after a prefix ``Images:``: ```python PromptInsertion( modality="image", target=PromptIndexTargets.prefix("Images:"), insertion="" * image_feature_size, ) ``` Insert these tokens at the end of the prompt: ```python PromptInsertion( modality="image", target=PromptIndexTargets.end(), insertion="" * image_feature_size, ) ``` Frepr insertionr>cC|jSrU)rryr;r;r<r{zPromptInsertion.contentcCtjSrU)rmrpryr;r;r<r~rzPromptInsertion.modeN) r4r5r6r7r rrr:rr{rmr~r;r;r;r<rs / rc@r) PromptReplacementa Defines how to replace portions of an input prompt with placeholder tokens. Example: For each image, replace one ```` input placeholder in the prompt with a number of ```` feature placeholders equal to the feature size of the vision encoder: ```python PromptReplacement( modality="image", target="", replacement="" * image_feature_size, ) ``` As above, but further pad the feature placeholders with ```` and ```, which are not supposed to be passed to the vision encoder: ```python PromptReplacement( modality="image", target="", replacement=PromptUpdateDetails( full="".join([ "", "" * image_feature_size, "", ]), features="" * image_feature_size, ), ) ``` To avoid unnecessary tokenization during prompt replacement, we recommended passing token sequences instead of text: ```python PromptReplacement( modality="image", target=[image_token_id], replacement=PromptUpdateDetails( full=([image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id]), features=[image_token_id] * image_feature_size, ), ) ``` Fr replacementr>cCrrU)rryr;r;r<r{\rzPromptReplacement.contentcCrrU)rmrqryr;r;r<r~`rzPromptReplacement.modeN) r4r5r6r7r rrr:rr{rmr~r;r;r;r<rs 4 ri)maxsizerMrKtextrNr>cCst|||dS)NrM)r)rKrrNr;r;r<_cached_encodeesrskip_special_tokensrd.rcCst|t||dS)Nr)rlist)rKrdrr;r;r<_cached_decodeqsrc@seZdZUeed<dS)_HasModalityAttrrsN)r4r5r6rPr:r;r;r;r<r}s rc@seZdZedefddZdS)_HasModalityPropr>cCsdSrUr;ryr;r;r<rsz_HasModalityProp.modalityN)r4r5r6rrPrsr;r;r;r<rsr_M)boundvaluescCst|dddS)z^Convenience function to apply [`full_groupby`][vllm.utils.full_groupby] based on modality.cSrrUrsxr;r;r<rEz'full_groupby_modality..key)r)rr;r;r<full_groupby_modalitysrc@seZdZUdZeddZeed<ee ed<ee e ed<e dede ddfd d Zdd d Zede fddZede e fddZd S)r[z A [`_PromptSeq`][vllm.multimodal.processing.PromptSeq] bound to a tokenizer to automatically convert between token sequence and text representations. FrrK_text _token_idsrJr>cCs.t|t|tr |ndt|tr|dSddS)N)rKrr)r[rOrPr)rKrJr;r;r<r^s z_BoundPromptSequence.from_seqNcCs$|jdur|jdurtddSdS)Nz8At least one of 'text' and 'token_ids' must be specified)rr ValueErrorryr;r;r< __post_init__sz"_BoundPromptSequence.__post_init__cCs2|jdur|jdus Jt|jt|j|_|jSrU)rrrrKtupleryr;r;r<rs z_BoundPromptSequence.textcCs2|jdur|jdus Jt|j|jdd|_|jSrL)rrrrKryr;r;r<rds z_BoundPromptSequence.token_idsr>N)r4r5r6r7r rKrr:rrPrr9rXr8r^rrrrdr;r;r;r<r[s$   r[c@s.eZdZUeed<eedgejfed<dS)_BoundPromptContentrZr[r\N) r4r5r6r[r:rrrargr;r;r;r<rs rc@seZdZUdZeed<eddZeed<ddd Z e de fd d Z e de eeffd d Ze defddZe defddZdedefddZdS)raK A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] bound to a tokenizer to automatically convert [`target`][vllm.multimodal.processing.PromptUpdate.target] and the result of [`get_content`][vllm.multimodal.processing.BoundPromptUpdate.get_content] between token sequence and text representations. rFrrKr>NcCstttf|_dSrU)dictr9r_content_cacheryr;r;r<rszBoundPromptUpdate.__post_init__cC|jjSrUrrsryr;r;r<rszBoundPromptUpdate.modalitycCs$|jj}t|tr |St|j|S)z'The token sequence (or text) to update.)rrtrOr2r[r^rK)rzrtr;r;r<rts zBoundPromptUpdate.targetcCrrv)rr{ryr;r;r<r{zBoundPromptUpdate.contentcCrr})rr~ryr;r;r<r~rzBoundPromptUpdate.modeitem_idxcCs||j}t|r|}||jvr|j|S||}nd}t|ts$t|}t|j|j}t ||j d}|dur<||j|<|S)z Given the index of the processed item within [`modality`][vllm.multimodal.processing.PromptUpdate.modality], output the token sequence (or text) to update. Nrf) r{callablerrOrYr^r[rKrZrr\)rzrr{ cache_keyZ bound_fullZ bound_contentr;r;r< get_contents$      zBoundPromptUpdate.get_contentr)r4r5r6r7rrr:r rKrrrrPrsrr[r2rtrr{rmr~r9rrr;r;r;r<rs   rc@seZdZUeed<eed<dS) _TokenMatch start_idxend_idxN)r4r5r6r9r:r;r;r;r<r s  r match_idsccs~t|}t|}|dkrdSd}|||dkr=||}||||kr/t||dV|}n|d7}|||dksdSdS)zh Yield each occurrence of `match_ids` in `token_ids`. Note that empty matches are ignored. rNr)rr)rQr)rdr prompt_lenZ match_lenrrr;r;r<iter_token_matchess rnew_idscCshttt}d}t||D]}|j}|j}|||||||}q|||dt|S)z} Replace each occurrence of `match_ids` in `token_ids` with `new_ids`. Note that empty matches are ignored. rN)rr9rrrappendr)rdrrout_seqs prev_end_idxmatchrrr;r;r<replace_token_matches,s  rFrc@sbeZdZUeed<edefddZeede fddZ eede fddZ defd d Z d S) PromptTargetMatchrr>cCrrUrryr;r;r<rsKrzPromptTargetMatch.modalitycCrurUrwryr;r;r<rOzPromptTargetMatch.start_idxcCrurUrwryr;r;r<rTrzPromptTargetMatch.end_idxcCs(t|jd|jd|jd|jdS)Nz (modality=z , start_idx=z , end_idx=))typer4rsrrryr;r;r<__repr__Ys zPromptTargetMatch.__repr__N) r4r5r6rr:rrPrsrr9rrrr;r;r;r<rGs rc@s:eZdZUeed<edefddZedefddZdS)_PromptTargetIndexMatchrSr>cCrrUrSryr;r;r<rbrz!_PromptTargetIndexMatch.start_idxcCrrUrryr;r;r<rfrz_PromptTargetIndexMatch.end_idxN)r4r5r6r9r:rrrr;r;r;r<r^ rc@s:eZdZUeed<edefddZedefddZdS)_PromptTargetTokenMatchrr>cCrrU)rrryr;r;r<rorz!_PromptTargetTokenMatch.start_idxcCrrU)rrryr;r;r<rsrz_PromptTargetTokenMatch.end_idxN) r4r5r6rr:rr9rrr;r;r;r<rkrrc@s@eZdZUejeed<edefddZ edefddZ dS)_PromptTargetTextMatchrr>cC |jSrU)rrHryr;r;r<r|r_z _PromptTargetTextMatch.start_idxcCrrU)rrWryr;r;r<rr_z_PromptTargetTextMatch.end_idxN) r4r5r6reMatchrPr:rr9rrr;r;r;r<rxs rc@s`eZdZUeed<eed<eed<eeed<eej ed<e defddZ de fd d Z d S) PlaceholderFeaturesInforsrrtokensr\r>cCs t|jSrU)rQrryr;r;r<lengthr_zPlaceholderFeaturesInfo.lengthcCst|j|j|jdS)N)offsetrr\)r(rrr\ryr;r;r<to_ranges z PlaceholderFeaturesInfo.to_rangeN)r4r5r6rPr:r9rrrargrrr(rr;r;r;r<rs  rrDprompt_updatesc$dtffdd fdd|DS)9Return each target of `prompt_updates` found in `prompt`.updatecsPj}t|tr|j}|durgSt|gSfddt|jDS)Ncg|]}t|qSr;)r.0rrr;r< z;find_token_matches..get_matches..)rtrOr2r3rKrrrdrrtrSrDrr< get_matchess    z'find_token_matches..get_matchescg|] }|D]}|qqSr;r;rrrrr;r<r z&find_token_matches..rrDrr;rrDr<find_token_matches rcr)rrcsXj}t|tr|j}|durgSt|gSfddtt|j DS)Ncrr;)rrrr;r<rrz:find_text_matches..get_matches..) rtrOr2r3rKrrfinditerescaperrrrr<rs   z&find_text_matches..get_matchescrr;r;rrr;r<rrz%find_text_matches..rrr;rr<find_text_matchesrr mm_matchesc Csdd|D}dgt|}|D](}t|j|jD]}||dur5td||d|d|d||||<qqt|dd d S) z Resolve `mm_matches` to ensure that there are no overlapping matches, and sort them such that earlier matches take priority over later ones. cSg|] }|D]}|qqSr;r;)rmatchesmr;r;r<rsz$_resolve_matches..NzFound overlapping matches (z and z ) at index=z of prompt=cSrrU)rrr;r;r<rErz"_resolve_matches..r)rrQrangerrrsorted)rDrrZ seen_matchesridxr;r;r<_resolve_matchess"  rmm_item_countscCsVtttttf}d}tttfdd}t||D]~}|j}||}||d} || kr0q|j} |j } |j } | j } | t j krM|||| | }n| t jkrd|||| | | kra| nd}nt| t||| }t||D]}| |}t|tr|jjn|jj}||qt| }||||7<q|||dttt|S).Apply the updates in `mm_matches` to `prompt`.rcSr@rAr;r;r;r;r<rErFz _apply_matches..rN)rrrPr9rrrsgetrrrr~rmrprrqrminrrrOrZrrdrr1)rDrrrrZnext_idx_by_modalityrrsZitem_start_idxZmax_item_countrroriginr~Z num_insertsZ item_end_idxrr{Z insert_seqr;r;r<_apply_matchess>     rcCs|s|St|||}t|S)r)rr)rDrrZ token_id_seqsr;r;r<apply_token_matchess rcCs|s|St|||}d|S)r)rjoin)rDrrZtextsr;r;r<apply_text_matches&s  rmm_prompt_updatesc cst|}tttfdd}d}||krd}|D]`\}}||} | ||dkr,q|D]H} | | } | jj} t| } || }| dksI||krJq.|||| krv| j }|dur^|| j}t || || |dV|}||d7<d}nq.|r{nq|s|d7}||ksdSdS) a& Yield each set of placeholder tokens found in `prompt`. Matches are exclusive even when multiple modalities share the same placeholder tokens. In that case, the modality that appears earlier in `mm_prompt_updates` takes priority. Note that empty matches are ignored. cSr@rAr;r;r;r;r<rECrFz$_iter_placeholders..rFN)rsrrrr\rT) rQrrPr9itemsrrrZrdr\r)rrDrrZitem_idx_by_modalityrfoundrsZmodality_updatesrZ update_infor{Zcontent_tokens_fullZcontent_len_fullZ end_idx_fullZcontent_is_embedr;r;r<_iter_placeholders4sN   rcCst|||}tt|SrU)rrr)rrDritr;r;r<find_mm_placeholdersqs  rcs&eZdZdeddffdd ZZS)ProcessingCache capacity_gbr>Ncs:t||t|_|jj|_|jj|_|jj|_dSrU) super__init__Z get_lru_cacher'_cacherputclearreset)rzr __class__r;r<r|s   zProcessingCache.__init__)r4r5r6floatr __classcell__r;r;r r<rzsrc seZdZdZdeddffdd ZedefddZde fd d Z dd d Z de ddfddZ edeeeeffddZdeeeffddZdedeeefdeeeeffddZZS)BaseProcessingInfozDBase class to provide the information necessary for data processing.ctxr>Ncst||_dSrU)rrr)rzrr r;r<rs  zBaseProcessingInfo.__init__cCs |jjjSrU)rZ model_configmodelryr;r;r<model_idr_zBaseProcessingInfo.model_idcCrrU)rrKryr;r;r< get_tokenizersz BaseProcessingInfo.get_tokenizerr-cCrrU)r get_hf_configryr;r;r<rs z BaseProcessingInfo.get_hf_configkwargsr/cKs|jjdi|S)zy Subclasses can override this method to handle specific kwargs from model config or user inputs. Nr;)rget_hf_processor)rzrr;r;r<rsz#BaseProcessingInfo.get_hf_processorcCru)z Return the maximum supported number of items for each modality. A value of `None` means unlimited number of items. Omitting a modality from the returned dictionary means that it is not supported at all. rwryr;r;r<get_supported_mm_limits z*BaseProcessingInfo.get_supported_mm_limitscCsZ|}|j}tttf}|D]\}}||}|dur#|nt||||<q|S)z=Return the maximum allowed number of items for each modality.N) rrZ get_mm_configrrPr9rZget_limit_per_promptr)rzsupported_mm_limitsZ mm_configZallowed_limitsrssupported_limitZ user_limitr;r;r<get_allowed_mm_limitss   z(BaseProcessingInfo.get_allowed_mm_limitsseq_len mm_countscCr@)a Return the maximum number of tokens per item of for each modality. When `None` (the default) is returned, vLLM will generate dummy inputs (images/videos) at maximum possible sizes and process them to determine the maximum token count per modality. This approach works but can be very slow for certain models (e.g., Qwen2.5-VL), leading to very long startup time. For better performance, each model can override this method to return pre-computed maximum token counts, avoiding the need for dummy input generation and processing. Note: The maximum number of tokens per item of each modality returned from this function should respect the model's maximum sequence length and the maximum number of items of each modality allowed, and agree with dummy inputs (images/videos) at maximum possible sizes. Nr;)rzrrr;r;r<get_mm_max_tokens_per_itemsz-BaseProcessingInfo.get_mm_max_tokens_per_item)r>r-)r4r5r6r7rrrrPrrrrobjectrrr rr9rrrrr;r;r r<rs$   r_IcseZdZdZdddedddeeddffd d Zed d Z ed dZ de de de e efdefddZdefddZde deddfddZde defddZeddde e efde e effddZed ede e efd!edeefd"d#Zd$e e eefd%eed&e e efde e ee ffd'd(Z!d ede"e e efe e efffd)d*Z#de de e efd+e e efd,e e efddf d-d.Z$d/e d ede e efd0e e efde%f d1d2Z&d/e d ede e efd0e e efde"eede%ff d3d4Z'd/e d0e e efdeefd5d6Z(d7eedeefd8d9Z)d ede e efd0e e efddfd:d;Z*de+e eefd ede e efd0e e efdZ,ded?ed@e-de"e.e ee/feffdAdBZ0d ede e efd0e e efde-fdCdDZ1dedEe.e ee/fdFede.e ee2ffdGdHZ3de+e eefd?ede e efd0e e efdIe%de"eeeee-e%ff dJdKZ4de+e eefd?ede e efd0e e efdIe%de"eeeee-e%ff dLdMZ5dNeede.e eeffdOdPZ6deedQe e ee7fd&e e efdeefdRdSZ8de dQe e ee7fd&e e efde fdTdUZ9dVeed$e e eefd&e e efde"eee e e ee fffdWdXZ:d+ed&e e efddfdYdZZ;d[e e ee fd&e e efddfd\d]ZZ?S)fBaseMultiModalProcessorz Abstract base class to process multi-modal inputs to be used in vLLM. Not to be confused with `transformers.ProcessorMixin`. N)cacheinfo dummy_inputszBaseDummyInputsBuilder[_I]r"r>csBt||_||_||_||_|j|_|j |_ dSrU) rrr#r$r"_get_data_parser data_parserr_supported_mm_limitsr_allowed_mm_limits)rzr#r$r"r r;r<rs   z BaseMultiModalProcessor.__init__cCrrU)r'ryr;r;r<rrz+BaseMultiModalProcessor.supported_mm_limitscCrrU)r(ryr;r;r<allowed_mm_limitsrz)BaseMultiModalProcessor.allowed_mm_limitsrDmm_datahf_processor_mm_kwargscCs||||SrU)apply)rzrDr*r+r;r;r<__call__sz BaseMultiModalProcessor.__call__cCstS)a Construct a parser to preprocess multi-modal data items before passing them to [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data]. You can support additional modalities by creating a subclass of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser] that has additional subparsers. )r,ryr;r;r<r%s z(BaseMultiModalProcessor._get_data_parserrs num_itemscCsh|j|d}|j|d}|dur|}t||}||kr2d|d|d}||kr.|d7}t|dS)NrzAt most  z"(s) may be provided in one prompt.z4 Set `--limit-mm-per-prompt` to increase this limit.)rrr)rr)rzrsr.rZ allowed_limitlimitmsgr;r;r<validate_num_itemss z*BaseMultiModalProcessor.validate_num_itemscCs2|j|}|D] \}}||t|q |S)a3 Normalize [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict] to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems] before passing them to [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data]. )r&Z parse_mm_datarr2rQ)rzr*mm_itemsrsrr;r;r< _to_mm_items*s z$BaseMultiModalProcessor._to_mm_items hf_inputsr.cCru)z?Given the HF-processed data, output the metadata of each field.rw)rzr5r+r;r;r<_get_mm_fields_config<sz-BaseMultiModalProcessor._get_mm_fields_configr3 out_mm_kwargscCru)aZ Given the original multi-modal items for this modality and HF-processed data, output the updates to perform. The information returned by this method is used to update token inputs which bypass the HF processor. It is also used to update the output of HF processor if the HF process does not apply prompt updates to text inputs. Moreover, this information is critical to determine the token positions in order to construct [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange] for each multi-modal item. rw)rzr3r+r7r;r;r<_get_prompt_updatesEsz+BaseMultiModalProcessor._get_prompt_updatesr new_token_idsrcC t|||SrU)r)rzrr9rr;r;r<_find_mm_placeholders\sz-BaseMultiModalProcessor._find_mm_placeholderscCsNtttf}tttf}|D]}||||q||fSrU)rrPrrrZget_processor_dataZget_passthrough_data)rzr3processor_datapassthrough_datarr;r;r<_get_hf_mm_dataes  z'BaseMultiModalProcessor._get_hf_mm_data mm_kwargs tok_kwargscCs<|jj|jjdi|tdd|i|tdi||S)zc Call the HF processor on the prompt text and associated multi-modal data. rNr;)r#rZcall_hf_processorrr)rzrDr*r?r@r;r;r<_call_hf_processorrs  z*BaseMultiModalProcessor._call_hf_processor prompt_texttokenization_kwargscCstdd|D S)z Return whether the HF processor applies prompt updates. For most HF processors, this should be `True` when multi-modal data items are passed, but `False` when multi-modal embeddings are passed. css|] }t|ttfVqdSrU)rOr*r)rrr;r;r< s  zHBaseMultiModalProcessor._hf_processor_applies_updates..)anyr)rzrBr3r+rCr;r;r<_hf_processor_applies_updatess z5BaseMultiModalProcessor._hf_processor_applies_updatesc CsV||\}}|j||||d}|||d\}|j||||d} ||| fS)z Apply the HF processor on the prompt text and multi-modal data together. In addition, return whether prompt updates have been applied. )rDr*r?r@Z input_idsrBr3r+rC)r>rArpoptolistrG) rzrBr3r+rCr<r=Zprocessed_data prompt_idsis_update_appliedr;r;r<_apply_hf_processor_text_mms    z3BaseMultiModalProcessor._apply_hf_processor_text_mmcCs |j|tii|d\}}}|S)z Apply the HF processor on the prompt text only. Since HF processor requires that text and multi-modal items correspond to each other, we create dummy multi-modal items to go along with the text. rH)rMr+)rzrBrCrK_r;r;r<_apply_hf_processor_text_onlys  z5BaseMultiModalProcessor._apply_hf_processor_text_only prompt_tokenscC|S)a Apply the HF processor on the prompt tokens only. Most HF processors accept prompt text but not prompt tokens. If the HF processor adds or removes tokens that are not related to multi-modal data, you should override this method so it is consistent with the output of [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only] on the corresponding text. r;)rzrPr;r;r<_apply_hf_processor_tokens_onlysz7BaseMultiModalProcessor._apply_hf_processor_tokens_onlycCs,|}|j|j||||d\}}}|S)aJ Apply the HF processor on the multi-modal data only. Since HF processor requires that text and multi-modal items correspond to each other, we generate dummy text using [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder] to go along with the multi-modal data. rH)get_all_countsrMr$Zget_dummy_text)rzr3r+rCrrNmm_processed_datar;r;r<_apply_hf_processor_mm_onlys  z3BaseMultiModalProcessor._apply_hf_processor_mm_onlyenable_hf_prompt_updatecCsRt|tr|r|j||||dS|||}n||}|j|||d}||dfS)a Apply the HF processor on the prompt text and multi-modal data. In addition, return whether prompt updates have been applied (for most HF processors, this should be `True`). Note: If `enable_hf_prompt_update=False`, we use HF processor to perform prompt updates if available; HF processor requires that the prompt corresponds to multi-modal items. rH)r3r+rCF)rOrPrMrOrRrU)rzrDr3r+rCrVrKrTr;r;r<_apply_hf_processor_mains$   z0BaseMultiModalProcessor._apply_hf_processor_main mm_data_items mm_hashescsNfdd|D}dd|D}fdd|D}|||fS)Ncs&i|]\}}|fdd|DqS)cs&g|]}|dur|nqSrUr)rhr"vr;r<r)sOBaseMultiModalProcessor._get_cache_missing_items...r;)rrshashesr\r;r< (s zDBaseMultiModalProcessor._get_cache_missing_items..cSs$i|]\}}|ddt|DqS)cSsg|] \}}t|tr|qSr;)rOrP)rr item_or_hashr;r;r<r/s r^) enumerate)rrsitems_or_hashesr;r;r<r`.s cs&i|]\}fdd|DqS)csg|]}|qSr;r;)rr)rXrsr;r<r6sr^r;)rZidxs)rXrr<r`5s)rr4)rzr"rXrYmm_cache_items_or_hashesZmm_missing_idxsZmm_missing_datar;)r"rXr]r<_get_cache_missing_items"s  z0BaseMultiModalProcessor._get_cache_missing_itemscs"|jjfdd|DS)z2Create MM hashes to be returned (only used in V1).cs*i|]\}fdd|DqS)cs,g|]}tjddi|iqS)rr;)r!Z hash_kwargsritem)r+rsrrCr;r<rFs zEBaseMultiModalProcessor._hash_mm_items...r;rDr+rrCrr<r`Es z:BaseMultiModalProcessor._hash_mm_items..)r#rr)rzr3r+rCr;rhr<_hash_mm_items<sz&BaseMultiModalProcessor._hash_mm_itemsrdmm_missing_kwargsc Cstttfdd}ttttft}|D].\}}|D]'}t|tr<||||} ||| ||d7<n|} || | qqt |S)NcSr@rAr;r;r;r;r<rEVrFz:BaseMultiModalProcessor._merge_mm_kwargs..r) rrPr9rr'rrOZget_itemrrr) rzr"rdrjZmm_missing_next_idxZ merged_itemsrsrcraZkw_itemr;r;r<_merge_mm_kwargsPs   z(BaseMultiModalProcessor._merge_mm_kwargsreturn_mm_hashesc CsP|j||||dd\}}}t||||} |r ||||nd} || | |fS)NTrDr3r+rCrV)rWr&from_hf_inputsr6ri) rzrDrXr+rCrlrKrTrLr?rYr;r;r<_apply_hf_processoris0    z+BaseMultiModalProcessor._apply_hf_processorcCs|j}||\}}|dus|r|j|||||dS||||} |j||| d\} } |r/| nd} |j|| ||dd\} }}t||||}|j || |d}tdd| D}| || |fS) zy Apply the HF processor on the full prompt text, caching the results and reusing cached results. N)rDrXr+rCrl)r"rXrYFrm)rdrjcSrr;r;)rZ cache_itemsrgr;r;r<rszFBaseMultiModalProcessor._cached_apply_hf_processor..) r"r>rorirerWr&rnr6rkr)rzrDrXr+rCrlr"rNr=rYrdZmm_missing_data_itemsZmm_hashes_to_returnrKZmm_missing_processed_datarLrjZmm_cache_items_mergedr?r;r;r<_cached_apply_hf_processors`      z2BaseMultiModalProcessor._cached_apply_hf_processorrcs(|jfdd|D}tt|S)Nc3s|]}|VqdSrU)r)rrrKr;r<rEszBBaseMultiModalProcessor._bind_and_group_updates..)r#rrr)rzrrr;rqr<_bind_and_group_updatess  z/BaseMultiModalProcessor._bind_and_group_updatesrcCr:rU)rrzrDrrr;r;r<_apply_token_matches z,BaseMultiModalProcessor._apply_token_matchescCr:rU)rrsr;r;r<_apply_text_matchesruz+BaseMultiModalProcessor._apply_text_matchesrdc s|j}fdd|D}dd|Dtfdd|Dr<|||t|dd|D}n't|fdd|D}|||t|dd d d|D}|||}|fS) Nci|] \}}|t|qSr;)rrrsZupdates)rdr;r<r` zABaseMultiModalProcessor._apply_prompt_updates..cSsi|] \}}|t|qSr;rV)rrsrr;r;r<r`sc3s$|] \}}|d|kVqdS)rNrZ)rrs item_count)mm_match_countsr;r<rEs  z@BaseMultiModalProcessor._apply_prompt_updates..cS i|] \}}|dd|DqS)cSg|]}|jqSr;rrr;r;r<rLBaseMultiModalProcessor._apply_prompt_updates...r;rrsZ token_matchesr;r;r<r`crwr;)rrx)rr;r<r`ryFrMcSr|)cSr}r;r~rr;r;r<r&rrr;rr;r;r<r`%r) r#rrallrtrrvrr;) rzrdrrrKZmm_token_matchesZmatched_updatesZmm_text_matches placeholdersr;)r{rrdr<_apply_prompt_updatessR       z-BaseMultiModalProcessor._apply_prompt_updatesc Csh|D]-\}}||jvr||}ng}t||kr1td|d|d|d|dt|d qdS)NExpected there to be r/z- items in keyword arguments corresponding to z data items, but only found z! There is likely a problem with your implementation of merged multi-modal processor for this model (usually arising from an inconsistency between `_call_hf_processor` and `_get_mm_fields_config`).)rZ modalitiesZ get_itemsrQ RuntimeError)rzr?rrsrzrr;r;r<_validate_mm_kwargs2s    z+BaseMultiModalProcessor._validate_mm_kwargsmm_placeholdersc CsT|D]#\}}||g}t||kr'td|d|d|dt|d qdS)Nrz! prompt updates corresponding to r/z items, but instead found z prompt updates! This is likely because you forgot to include input placeholder tokens (e.g., ``, `<|image_pad|>`) in the prompt. If the model has a chat template, make sure you have applied it before calling `LLM.generate`.)rrrQr)rzrrrsrzrr;r;r<_validate_mm_placeholdersGs  z1BaseMultiModalProcessor._validate_mm_placeholdersrKrLc Cs||||}||}|}||||r0||||} || ||j} t| |} n| |||\}} } || ||| | fSrU) r8rrrSrr;rr#rrr) rzr3r+rKr?rLZunbound_prompt_updatesrrrrKrDr;r;r<_maybe_apply_prompt_updates\s<      z3BaseMultiModalProcessor._maybe_apply_prompt_updatesFc Csr||}|dur i}|j|||||d\}}} } |j||||| d\}}} dd| D} td|||| | dS)a" Process multi-modal inputs to be used in vLLM. The main steps are: 1. Apply HF Processor on prompt text and multi-modal data together, outputting token IDs and processed tensors. 2. Find and update sequences in the token IDs with placeholder tokens. The number of placeholder tokens equals the feature size of the multi-modal data outputted by the multi-modal encoder. 3. Extract information about the placeholder tokens from the processed token IDs. N)rCrl)r3r+rKr?rLcSr|)cSsg|]}|qSr;)rrfr;r;r<rsz..r;)rrsrr;r;r<r`rz1BaseMultiModalProcessor.apply..Z multimodal)rrDprompt_token_idsr?rYr)r4rprrr%) rzrDr*r+rCrlr3rKr?rYrLrZmm_placeholder_rangesr;r;r<r,sB   zBaseMultiModalProcessor.applyNF)@r4r5r6r7r rrrrrr)rPr"r rr%r-r,r%r9r2r+r4rr$r6r&r rrr8rrrr;rr>rAboolrGrMrOrRrUrrWMultiModalHashesr_CacheItemOrHashrerir'rkrorprrrrtrvrrrrr,rr;r;r r<r!s                            "         *        !   H        G      0 r!c seZdZedeeeefdedeeeeffddZ e de fddZ deeeefdedeeeeffdd Z deeeefded efd d Z ddeeeefdedeeefdeeeefde def fdd ZZS)EncDecMultiModalProcessorrDr*r>cCru)z Create input prompt for the encoder. HF processor will be applied on this prompt during profiling and generation. rwrzrDr*r;r;r<create_encoder_promptrz/EncDecMultiModalProcessor.create_encoder_promptcCr@rr;ryr;r;r<pad_dummy_encoder_promptrz2EncDecMultiModalProcessor.pad_dummy_encoder_promptcCrQ)z$Create input prompt for the decoder.r;rr;r;r<create_decoder_promptsz/EncDecMultiModalProcessor.create_decoder_promptencoder_inputscCsn|j}|||}t|trt||dd}n|}t||}td|d|dd|}|||d|S)NFrMrDr)encoder_promptZencoder_prompt_token_ids)rDrr;) r#rrrOrPrrr#r)rzrDr*rrKZdecoder_promptZdecoder_prompt_idsZ mm_inputsr;r;r<_get_enc_dec_inputss(    z-EncDecMultiModalProcessor._get_enc_dec_inputsNFr+rCrlcs0|||}t|||||}|j|||dS)a; Process multi-modal inputs to be used in vLLM. The main processing steps are modified to fit encoder-decoder model: 1. Create encoder prompt from input prompt text. 2. Apply the HF processor on encoder prompt. 3. Copy the input prompt text as decoder prompt inputs. )rDr*r)rrr,r)rzrDr*r+rCrlrrr r;r<r,s zEncDecMultiModalProcessor.applyr)r4r5r6rrrPrr9r"rrrrrr%rr rrr#r,rr;r;r r<rsP     r)yabcrr collectionsrcollections.abcrrrrr r dataclassesr r enumr functoolsrtypingrrrrrrrrregexrraZtyping_extensionsrZ vllm.inputsrZ vllm.loggerrZ!vllm.transformers_utils.tokenizerrrrZ vllm.utilsrrr"r Zhasherr!Zinputsr"r#r$r%r&r'r(parser)r*r+r,Z transformers.configuration_utilsr-Z%transformers.feature_extraction_utilsr.Ztransformers.processing_utilsr/Z profilingr0r4loggerrPrr9r1r8r2r=rrYZPromptUpdateInforrmrrrrrrrrrrrrr[rrrrrrrrrrrrrrrrrrrrrr rrr!rr;r;r;r<sr    (     $     1 1 DI     ".D          1      =    Nk