o )iX@sddlmZmZddlmZddlmZddlmZm Z m Z m Z er(ddl m Z ddlmZmZe dZGd d d ZGd d d ee eZd S))ABCabstractmethod)Sequence)Path) TYPE_CHECKINGGeneric NamedTupleTypeVar)SequenceGroupMetadata)MultiModalKwargsPlaceholderRange_Tc @seZdZUdZGdddeZeeed< e ed< eeed< e ed< dd Z e d d d ed e e eedfffddZd edeedeed eefddZdddZdddZdS)MultiModalPlaceholderMapzq Relates multi-modal embeddings to their corresponding placeholders. Note: This is only used in V0. c@s&eZdZUeeed<eeed<dS)z!MultiModalPlaceholderMap.IndexMapsrcdestN)__name__ __module__ __qualname__listint__annotations__rr`/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/vllm/multimodal/base.pyIndexMaps r src_rangessrc_len dest_rangesdest_lencCsg|_d|_g|_d|_dS)Nr)rrrrselfrrr__init__2s z!MultiModalPlaceholderMap.__init__ seq_groupr positionsreturnc Csp|j}|j}|r |stifStttf}|D]\}}t}|r/||dgt|||||<q||fS)a Returns the multi-modal items that intersect with the portion of a prompt (``seq_group``) represented by ``positions``, as well as a ``MultiModalPlaceholderMap`` that relates the multi-modal embedding vectors to their corresponding placeholders. Examples: ``` Prompt: |AAAA BBBB What's in these images?| Positions: |.................................| images = [A, B] src_ranges = [(0, 4), (4, 8)] dest_ranges = [(0, 4), (5, 9)] Prompt: |AAAA BBBB What's in these images?| Positions: | ..... | images = [A, B] src_ranges = [(2, 4), (4, 6)] dest_ranges = [(0, 2), (3, 5)] Prompt: |AAAA BBBB What's in these images?| Positions: | ......... | images = [B] src_ranges = [(0, 4)] dest_ranges = [(0, 4)] Prompt: |AAAA BBBB What's in these images?| Positions: | .......................| images = [] src_ranges = [] dest_ranges = [] ``` N) Zmulti_modal_datamulti_modal_placeholdersr dictstrritemsappend_items_from_seq_grouplen) clsr"r#Z seq_mm_dataZseq_mm_placeholdersZplaceholder_mapsZmodalityZ placeholdersZplaceholder_maprrrfrom_seq_group8s*   z'MultiModalPlaceholderMap.from_seq_groupmulti_modal_itemsr%c Csg}t|t|krtdt||D]Z\}}t|j|j|j}tt|j|jt|j |j }|s3qt|j|j|j |j} t|j|j|j |j |j|j } | ||j | |j | |j t|7_ q|jt|7_|S)z Adds the multi-modal items that intersect ```positions`` to this placeholder map and returns the intersecting items. z=Multi-modal placeholders and items must have the same length.)r* ValueErrorziprangeoffsetlengthmaxstartminstoprappendrrr) r r#r-r%Zintersecting_itemsZplaceholder_dictZmm_item placeholder intersectionZtoken_embedding_rangeZmultimodal_embedding_rangerrrr)ys@          z4MultiModalPlaceholderMap.append_items_from_seq_groupothercs\jfdd|jDj|j7_jfdd|jDj|j7_dS)z Adds the placeholders from another ``MultiModalPlaceholderMap`` to this instance based on the source and destination tensors being concatenated. c3*|]}tj|jj|jVqdSN)r0rr4r6.0rrrr  z2MultiModalPlaceholderMap.extend..c3r;r<)r0rr4r6r=rrrr@rAN)rextendrrr)r r:rrrrBszMultiModalPlaceholderMap.extendcCs\dd|jD}dd|jD}t|t|kr'tdt|dt|d|j||dS)z Finalizes the placeholder map into lists of indices that can be used to index the source and destination tensors. cSg|] }|D]}|qqSrrr>r?irrr z6MultiModalPlaceholderMap.index_map..cSrCrrrDrrrrFrGzThe number of source (z) and destination indices (z) must be the same.)rr)rrr*r.r)r Z src_indicesZ dest_indicesrrr index_maps z"MultiModalPlaceholderMap.index_mapN)r:r)r$r)rrr__doc__rrrr0rrr! classmethodtupler r&r'r,rrr r)rBrHrrrrrs>    @ 1rc@sReZdZededefddZedededefddZede defd d Z d S) MediaIOdatar$cCtr<NotImplementedError)r rMrrr load_byteszMediaIO.load_bytes media_typecCrN)zm List of media types: https://www.iana.org/assignments/media-types/media-types.xhtml rO)r rSrMrrr load_base64szMediaIO.load_base64filepathcCrNr<rO)r rUrrr load_filerRzMediaIO.load_fileN) rrrrbytesrrQr'rTrrVrrrrrLsrLN)abcrrcollections.abcrpathlibrtypingrrrr Z vllm.sequencer Zinputsr r rrrLrrrrs   ;