o
    rqiX                    @   s  d dl Z d dlZd dlZd dlmZ d dlmZmZmZm	Z	m
Z
mZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d	d
lmZmZ d	dlmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( e Z)dZ*e	eee+e+f e	e+ f  Z,e	ee+e	e- e	e+ f  Z.e.Z/ee+e	e- f Z0G dd dZ1ddde+dede2ddfddZ3e3e1j4eg dgdgdgge*dgdd G dd deZ5e3e1j6e5 dd e3e1j7ed ggd!gddggdd G d"d# d#Z8G d$d% d%e8eZ9G d&d' d'e8Z:G d(d) d)e:eZ;G d*d+ d+Z<e3e1j=e;  G d,d- d-e<e;Z>G d.d/ d/e<e5Z?e3e1j@e>  e3e1jAe?  e3e1jBe9  e3e1jCeg d0gg d1ge*d2g G d3d4 d4ZDG d5d6 d6eDe;ZEG d7d8 d8eDe5ZFe3e1jGeE dd9 e3e1jHeF ddd: G d;d< d<ZIG d=d> d>eIe;ZJG d?d@ d@eIe5ZKe3e1jLeJ dd9 dAdB ZMG dCdD dDe;ZNe3e1jOeN dd9 e3e1jPeK ddd: G dEdF dFe9ZQe3e1jReQ  dGZSG dHdI dIeZTG dJdK dKeZUG dLdM dMeUZVe3e1jWeV dNdddO e3e1jXeTg dPgdQdPgdRgdSgdSdPggeSdgddTddU e3e1jYedVgdWgd!dXggg dgg e3e1jZeUdVgdYgdgdgg e3e1j[eUg d!gddggdd e3e1j\eUg dZgg d[gdd\g e3e1j]eUg dZgg d[gdd\gd]d^gd_ d`Z^e3e1j_eUg dZgg dage^d\g e3e1j`ed ggdbgdggdggdd gdg e3e1jaed ggdcgdSgdggdd gdVg e3e1jbeddggdbgdeggdeggdddgdg e3e1jcedfgdggdhgdhgddig djZde3e1jeedkgdlgdmgdngeddog e3e1jfedpgdqgdpgdrgddsg e3e1jgedkgddqgdtgdng G dudv dvZhG dwdx dxeheZiG dydz dzeheZje3e1jkej  e3e1jlei  d{Zme3e1jneg d|gdSgdggemdgdd d}Zoe3e1jpeg d~gdgdgeodgdd dZqe3e1jredgdgdgdgeqdg edZset Zude+dee+gesf dees dees fddZvG dd de9Zwe3e1jxew  	d/de+de,de+dee+e,e	e+ f fddZyG dd deZze3e1j{ezddddd G dd dezZ|e3e1j}e|ddddd e3e1j~e|ddddd G dd deZde	e0 fddZG dd deZG dd dZG dd deeZG dd deeZe3e1je dddNd e3e1je dddNd e3e1je ddd e3e1je ddd G dd deZe3e1je dddNdd e3e1jedVgdgdggdgg e3e1jeg dgddgg e3e1jed gdVgdgdSgdgg e3e1jedgdgddg e3e1jed gdVgdgg dgg e3e1jedVgdgg dgg e3e1jeg dgdgdSgdggddg e3e1jedVgdgdgdgd G dd deZG dd deZe3e1je ddd G dd deZe3e1jedgdgdgdngddd e3e1jedgdgdgdgddTddU de	e- de	e- de-dee	e- e	e- f fdd˄ZG dd̈́ deZe3e1jedgdgdgdgddgddd G ddԄ deZe3e1je ddd G ddք deZG dd؄ deZG ddڄ deZG dd܄ deZe3e1je ddd e3e1je ddd G ddބ deZe3e1je ddd G dd deheZe3e1je ddd G dd de:eZe3e1je ddd G dd de:eZe3e1je ddd G dd deiZe3e1je ddd G dd deZe3e1je dNddd G dd deZe3e1je  G dd deZe3e1je dd9 G dd deheZe3e1je ddd G dd de:eZe3e1je ddd de	e- dee-e	e- f de	e- fddZG dd deZe3e1je ddd e3e1jeg dgdgdngddg e3e1jedVgdgdagdag e3e1jedgdgdngdng G dd deZe3e1jedgdgg dngddNddU e3e1jedgd gddngddNddU e3e1jed ggdgdSgdggddNddU G dd deZe3e1jed ggdgdSgdggddNddd e3e1jedgdgg dng de	e- de	e- de	e- fd	d
ZG dd deZG dd de:eƃZe3e1jeǃ ddd G dd deheƃZe3e1jeɃ dddNd e3e1jedgdgg dngdddNd edgdgdgdgddgZe3e1jẽ e3e1jeg dgdgdg e3e1jeg dgg dg dZG dd de9Ze3e1jeу  e3e1jeg dgg dggddg dZe3e1jedgd gd!gd!geԐd"g G d#d$ d$eZe3e1jeփ dTddd% G d&d' d'e:eZe3e1je؃ ddd e3e1jedVgd(gdgdng d)Ze3e1jedVgdgdngdngeۃ e3e1jedVgd*gdngdng G d+d, d,ZG d-d. d.ZdS (0      N)partial)	AnyCallableDictListLiteralOptionalTupleTypeVarUnion)version)PreTrainedTokenizerBase)get_class_from_dynamic_module)is_deepspeed_zero3_enabled)
get_logger   )TemplateTEMPLATE_MAPPING)load_audio_qwen
load_batch
load_imageload_video_cogvlm2load_video_internvlload_video_llavaload_video_minicpmv_mplug_owl3load_video_qwen2transform_imageupper_bound	fetch_oneYou are a helpful assistant.c                   @   s  e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zd
ZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!d Z"d!Z#d"Z$d#Z%d$Z&d%Z'd&Z(d'Z)d(Z*d)Z+d*Z,d+Z-d,Z.d-Z/d.Z0d/Z1d0Z2d1Z3d2Z4d3Z5d4Z6d5Z7d6Z8d7Z9d8Z:d9Z;d:Z<d;Z=d<Z>d=Z?d>Z@d?ZAd@ZBdAZCdBZDdCZEdDZFdEZGdFZHdGZIdHZJdIZKdJZLdKZMdLZNdMZOdNZPdOZQdPZRdQZSdRZTdSZUdTZVdUZWdVZXdWZYdXZZdYZ[dZZ\d[Z]e^d\e_e` fd]d^Zad_S )`TemplateTypezdefault-generationzchatglm-generationzqwen-vl-generationzqwen-audio-generationdefaultqwenzqwen-vlz
qwen-audiozqwen2-audiozqwen2-audio-generationzqwen2-vlzmodelscope-agentbaichuanchatglm2chatglm3chatglm4	codegeex4llamallama3
reflectionzlongwriter-llama3llava1_5zllava-mistralzllava-vicunazllava-yizllama-llava-next-hfzllava-next-llama3zllama-qwen-hfzllava-onevision-qwenzllava-next-videozllava-next-video-yizllama3-llava-nextz
llava-qwenzllava-llama-instructidefics3zmistral-nemo	openbuddy
openbuddy2internlm	internlm2zinternlm-xcomposer2zinternlm-xcomposer2-4khdzinternlm-xcomposer2_5internvl	internvl2zinternvl-phi3zinternvl2-phi3florencezyi-coderzyi-vlyuanxverseziyaskyworkbluelmzephyrsusdeepseekznumina-mathzdeepseek-coderzdeepseek-vl	deepseek2deepseek2_5zcodefuse-codellamacodefusecogvlmzcogvlm2-videoglm4vzcogagent-chatzcogagent-instructorionminicpmz	minicpm-vzminicpm-v-v2_5zminicpm-v-v2_6gemma	paligemmaz
mplug-owl2
mplug_owl3zwizardlm2-awq	wizardlm2atomphi3zphi3-vltelechatztelechat-v2dbrxmengzic4aichatmlzdefault-generation-bosreturnc                 C   s<   g }| j  D ]}|ds|dkrq|| j |  q|S )N__get_template_name_list)__dict__keys
startswithappend)clsresk rX   r/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/modelscope/preprocessors/templates/template.pyrP      s   z#TemplateType.get_template_name_listN)b__name__
__module____qualname__default_generationchatglm_generationqwen_vl_generationqwen_audio_generationr!   r"   qwen_vl
qwen_audioqwen2_audioqwen2_audio_generationqwen2_vlmodelscope_agentr#   r$   r%   r&   r'   r(   r)   r*   longwriter_llama3r+   llava_mistralllava_vicunallava_yillama3_llava_next_hfllava_next_llama3llava_qwen_hfllava_onevision_qwenllava_next_videollava_next_video_yillama3_llava_next
llava_qwenllava_llama_instructr,   mistral_nemor-   r.   r/   r0   internlm_xcomposer2internlm_xcomposer2_4khdinternlm_xcomposer2_5r1   r2   internvl_phi3internvl2_phi3r3   yi_coderyi_vlr4   r5   r6   r7   r8   r9   r:   r;   numina_mathdeepseek_coderdeepseek_vlr<   r=   codefuse_codellamar>   r?   cogvlm2_videor@   cogagent_chatcogagent_instructrA   rB   	minicpm_vminicpm_v_v2_5minicpm_v_v2_6rC   rD   
mplug_owl2rE   wizardlm2_awqrF   rG   rH   phi3_vlrI   telechat_v2rJ   rK   rL   rM   default_generation_bosclassmethodr   strrP   rX   rX   rX   rY   r       s    r    F)exist_oktemplate_typetemplater   rN   c                K   s:   |s| t v rtd|  d| |_d|i|}|t | < d S )NzThe `z6` has already been registered in the TEMPLATE_MAPPING.r   )r   
ValueErrorr   )r   r   r   kwargsZtemplate_inforX   rX   rY   register_template   s
   r   z%### Human:
{{QUERY}}

### Assistant:
z

eos_token_idz{{SYSTEM}}

Tauto_add_bosc                          e Zd Z fddZ  ZS )DefaultGenerationTemplatec                    s    t  jg dgd dggdd d S )N	{{QUERY}}r   Tr   super__init__self	__class__rX   rY   r          z"DefaultGenerationTemplate.__init__rZ   r[   r\   r   __classcell__rX   rX   r   rY   r          r   )is_generationZbos_token_idr   c                   @   s    e Zd ZdZddefddZdS )ChatmlTemplateMixinNTr   c              
   C   s(   t j| g dgdgdg| jdg|d d S )Nz;<|im_start|>user
{{QUERY}}<|im_end|>
<|im_start|>assistant
z<|im_end|>

<|im_end|>z(<|im_start|>system
{{SYSTEM}}<|im_end|>
r   r   r   system)r   r   rX   rX   rY   r      s   
zChatmlTemplateMixin.__init__)T)rZ   r[   r\   r   boolr   rX   rX   rX   rY   r          r   c                   @      e Zd ZdS )ChatmlTemplateNrZ   r[   r\   rX   rX   rX   rY   r          r   c                       s    e Zd ZeZ fddZ  ZS )QwenTemplateMixinc                    s   t  jdd d S )NFr   r   r   r   rX   rY   r      s   zQwenTemplateMixin.__init__)rZ   r[   r\   DEFAULT_SYSTEMr   r   r   rX   rX   r   rY   r      r   r   c                   @   r   )QwenTemplateNr   rX   rX   rX   rY   r      r   r   c                	   @   s   e Zd ZdZdd Zded dedeee	f de
e fd	d
Zdedeee	f de
e fddZdedeee	f de
e fddZdS )_QwenVLTemplateMixinFc                 C   s0   | dpg }|rtt|tsJ dd S d S )Nimagesz0QwenVL only supports datasets with images paths!)get
isinstancer   r   r   exampler   rX   rX   rY   check_example   s   "z"_QwenVLTemplateMixin.check_example
media_typeimagevideoaudioindexr   rN   c                 C   H   |dksJ | dpg }|| }t|tsJ d|d  d| dgS )Nr   r   zPicture r   z:<img></img>
r   r   r   )r   r   r   r   r   r   rX   rX   rY   replace_tag   
   z _QwenVLTemplateMixin.replace_tagc                 C   s"   |d }|| }d|d  dgS )Nobjects<ref>caption</ref>rX   r   r   r   r   object_rX   rX   rY   replace_object   s   z#_QwenVLTemplateMixin.replace_objectc                 C   s   |d }|| }t |d d tr6d}|d D ]}|d|d  d|d  d|d	  d|d
  d	7 }q|gS d|d d  d|d d  d|d d	  d|d d
  d	gS )Nr   bboxr    z<box>(,r   ),(      z)</box>r   listr   r   r   r   r   all_objects
sub_objectrX   rX   rY   replace_box   s   4

z _QwenVLTemplateMixin.replace_boxN)rZ   r[   r\   Zload_mediasr   r   intr   r   r   r   Contextr   r   r   rX   rX   rX   rY   r      s    

"&r   c                   @   r   )QwenVLTemplateNr   rX   rX   rX   rY   r      r   r   c                   @   r   )QwenVLGenerationTemplateNr   rX   rX   rX   rY   r      r   r   z& 

<|user|>:{{QUERY}} 

<|assistant|>:z 

</s>z 

<|system|>:{{SYSTEM}}c                	   @   s   e Zd Zded dedeeef dee	 fddZ
deeef deeeef eeef f fdd	Zd
edeeef fddZdeeef deeef ddfddZddeeeef  dee deeef fddZdS )_QwenAudioTemplateMixinr   r   r   r   rN   c                 C   r   )Nr   audiosAudio r   z:<audio>z	</audio>
r   )r   r   r   r   r   r   rX   rX   rY   r      r   z#_QwenAudioTemplateMixin.replace_tagc                 C   sB   t | |\}}t|dkr||fS |dd  || ||fS )Nr   
loss_scale)r   _encodelenpopupdate)r   r   inputstokenizer_kwargsrX   rX   rY   r     s   
z_QwenAudioTemplateMixin._encodecontextc                 C   s   d| j |iS )N
audio_info)	tokenizerZprocess_audio)r   r   rX   rX   rY   _get_tokenizer_kwargs     z-_QwenAudioTemplateMixin._get_tokenizer_kwargsr   curr_tokenizer_kwargsNc                 C   s~   | d}| d}|d u r||d< d S |d ur;dD ]}tj|| || gdd||< qdD ]}|| ||  ||< q.d S d S )Nr   )Zinput_audiosZinput_audio_lengthsr   dim)Zaudio_span_tokensZ
audio_urls)r   torchconcat)r   r   r   r   Zold_audio_inforW   rX   rX   rY   _concat_tokenizer_kwargs  s   

 z0_QwenAudioTemplateMixin._concat_tokenizer_kwargsbatch
padding_toc                 C   s6   t | ||}|d dd urdd |D |d< |S )Nr   r   c                 S      g | ]}|d  qS )r   rX   .0brX   rX   rY   
<listcomp>       z9_QwenAudioTemplateMixin.data_collator.<locals>.<listcomp>)r   data_collatorr   )r   r   r   rV   rX   rX   rY   r     s   z%_QwenAudioTemplateMixin.data_collatorN)rZ   r[   r\   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   rX   rX   rX   rY   r      s    

2&4r   c                   @   r   )QwenAudioTemplateNr   rX   rX   rX   rY   r   $  r   r   c                   @   r   )QwenAudioGenerationTemplateNr   rX   rX   rX   rY   r   (  r   r   )lazy_tokenize)r   r   c                	   @   sn   e Zd Zdeeef deeeef eeef f fddZd
deeeef  de	e
 deeef fdd	ZdS )_Qwen2AudioTemplateMixinr   rN   c                 C   s   t | |\}}t|dkr|i fS | jj}|jj}t|dp!g t	t
|dd}|r@|j||ddd}|d|d	< || |i fS )
Nr   r   )sampling_rate)Z	load_funcTpt)r   Zreturn_attention_maskreturn_tensorsZattention_maskfeature_attention_mask)r   r   r   r   	processorZfeature_extractorr   r   r   r   r   r   r   )r   r   r   _r  r   r   Zaudio_inputsrX   rX   rY   r   3  s   
z _Qwen2AudioTemplateMixin._encodeNr   r   c                 C   sN   t | ||}dd |D }|r%t||d< dd |D }t||d< |S )Nc                 S   s"   g | ]}| d dur|d  qS )input_featuresNr   r   rX   rX   rY   r   D     " z:_Qwen2AudioTemplateMixin.data_collator.<locals>.<listcomp>r  c                 S   r   )r  rX   r   rX   rX   rY   r   G  r   r  )r   r   r   r   )r   r   r   rV   r  r  rX   rX   rY   r   B  s   z&_Qwen2AudioTemplateMixin.data_collatorr   )rZ   r[   r\   r   r   r   r	   r   r   r   r   r   rX   rX   rX   rY   r   1  s    24r   c                	   @   6   e Zd Zded dedeeef dee	 fddZ
dS )	Qwen2AudioTemplater   r   r   r   rN   c                 C   s   |dksJ d|d  dgS )Nr   r   r   z&: <|audio_bos|><|AUDIO|><|audio_eos|>
rX   r   r   r   r   rX   rX   rY   r   N  s   zQwen2AudioTemplate.replace_tagNrZ   r[   r\   r   r   r   r   r   r   r   r   rX   rX   rX   rY   r  L      
r  c                	   @   r  )	Qwen2AudioGenerationTemplater   r   r   r   rN   c                 C      |dksJ dgS )Nr   z$<|audio_bos|><|AUDIO|><|audio_eos|>
rX   r	  rX   rX   rY   r   V  s   z(Qwen2AudioGenerationTemplate.replace_tagNr
  rX   rX   rX   rY   r  T  r  r  c                 C   s   ddl m}m}m}m} tdt|}tdtd }tdtd }|r,|r,||||d\}}n| j\}}	tdt|}
tdt|}||	|||
|d	\}}| ||f} | S )
Nr   )IMAGE_FACTOR
MIN_PIXELS
MAX_PIXELSsmart_resizesize_factorresized_heightresized_width)factor
min_pixels
max_pixels)r  r  r  )	Zqwen_vl_utils.vision_processr  r  r  r  get_env_argsr   sizeresize)r   r  r  r  r  r  r  r  widthheightr  r  rX   rX   rY   _process_image_qwen_  s,   

r  c                	       s   e Zd Zded dedeeef dee	 fddZ
dedeeef dee	 fdd	Zdedeeef dee	 fd
dZdeeef deeeef eeef f f fddZddeeeef  dee deeef f fddZ  ZS )Qwen2VLTemplater   r   r   r   rN   c                 C   s    |dv sJ |dkrdgS dgS )N>   r   r   r   z+<|vision_start|><|image_pad|><|vision_end|>z+<|vision_start|><|video_pad|><|vision_end|>rX   r	  rX   rX   rY   r   |  s   zQwen2VLTemplate.replace_tagc                 C   s*   | d}|r|| }d|d dgS dgS )Nr   z<|object_ref_start|>r   z<|object_ref_end|><ref-object>r  r   rX   rX   rY   r     s
   
zQwen2VLTemplate.replace_objectc                 C   s   | d}|rY|| }t|d d tr9d}|d D ]}|d|d  d|d  d|d	  d|d
  d	7 }q|gS d|d d  d|d d  d|d d	  d|d d
  d	gS dgS )Nr   r   r   r   z<|box_start|>(r   r   r   r   r   z)<|box_end|><bbox>r   r   r   r   rX   rX   rY   r     s&   


zQwen2VLTemplate.replace_boxc                    s  t  |\}}t|dkr|i fS | jj}|d }|d }|dp$g }|dp+g }dD ]}	t |	 r|	dkrMt|t}d}
|j	|d dd	}|d
 }nt|t
}|j	d |dd	}|d }d}
t||
}d}t|D ]H\}}|j	jd }||  | }|d ||  |
g|  ||| d d   }|r|d ||  dg|  ||| d d   }||d 7 }qk|| q.||d< ||d< |i fS )Nr   	input_idslabelsr   videos)r   r$  igP r   )r   r$  r   Zimage_grid_thwZvideo_grid_thwihP r   r   )r   r   r   r   r  r   localsr   r  image_processorr   _findall	enumerateZ
merge_sizeprodr   )r   r   r   r  r  r"  r#  r   r$  r   Zmedia_tokenZmedia_inputsZmedia_grid_thwidx_listadded_tokens_leniidxZmerge_lengthZ	token_lenr   rX   rY   r     sZ   




 
zQwen2VLTemplate._encodeNr   r   c                    sF   t  ||}dD ]  fdd|D }|r t||  d< q	|S )N)r   r   c                    s.   g | ]}|   d dur|  d  qS )	_grid_thwNr  r   r   rX   rY   r     s   . z1Qwen2VLTemplate.data_collator.<locals>.<listcomp>r/  r   r   r   r   )r   r   r   rV   Zgrid_thwr   r0  rY   r     s   zQwen2VLTemplate.data_collatorr   )rZ   r[   r\   r   r   r   r   r   r   r   r   r   r   r	   r   r   r   r   rX   rX   r   rY   r  z  s    

""6<'r  c                   @      e Zd ZdZdS )YiCoderTemplater   NrZ   r[   r\   r   rX   rX   rX   rY   r3        r3  u  This is a chat between an inquisitive human and an AI assistant. Assume the role of the AI assistant. Read all the images carefully, and respond to the human's questions with informative, helpful, detailed and polite answers. 这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。仔细阅读所有的图像，并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。c                	       s   e Zd Zdee fddZdeeef de	eeef eeef f f fddZ
ddeeeef  d	ee deeef f fd
dZ  ZS )YiVLTemplaterN   c                 C      |dksJ dgdgS Nr   8
rX   r	  rX   rX   rY   r        
zYiVLTemplate.replace_tagr   c                    s   t  |\}}t|dkr|i fS |dd  ddlm} | jj}|dp(g }t	|D ]\}}	t
dd |jD }
||	|
}	|	||< q-|rY|j|ddd	 }||d
 |d< |i fS )Nr   r   )expand2squarer   c                 s   s    | ]	}t |d  V  qdS )   N)r   r   xrX   rX   rY   	<genexpr>  s    z'YiVLTemplate._encode.<locals>.<genexpr>r   r   pixel_valuesdtype)r   r   r   r   llava.mm_utilsr<  r   r'  r   r)  tupleZ
image_mean
preprocessto)r   r   r   r   r  r<  r'  r   r-  r   background_colorZimage_tensorr   rX   rY   r     s   

zYiVLTemplate._encodeNr   r   c                    sp   t  ||}dd |D }|rt||d< dd |d D }tdd |D s6tdd |D r6J d|S )	Nc                 S      g | ]
}d |v r|d  qS r   rX   r   rX   rX   rY   r         z.YiVLTemplate.data_collator.<locals>.<listcomp>r   c                 S      g | ]}|d k  qS r9  sumr   rX   rX   rY   r         r"  c                 S      g | ]}|d kqS r   rX   r   hrX   rX   rY   r          c                 S   rQ  rR  rX   rS  rX   rX   rY   r         zCYIVL does not support mix-batch nlp dataset and multi-modal dataset)r   r   r   r   allanyr   r   r   rV   r   Z
has_imagesr   rX   rY   r     s   zYiVLTemplate.data_collatorr   )rZ   r[   r\   r   r   r   r   r   r   r	   r   r   r   r   r   rX   rX   r   rY   r6    s    6<r6  c                       s&   e Zd Zdeddf fddZ  ZS )GLMTemplater   rN   Nc                    sN   t  j|g|R i |}|d}| jd| | jd ur%| jd| |S )Nr   r   )r   _init_templateencodeprefixinsertsystem_prefix)r   r   argsr   rV   
token_listr   rX   rY   r[  	  s   

zGLMTemplate._init_template)rZ   r[   r\   r   r[  r   rX   rX   r   rY   rZ    s    rZ  c                	       s   e Zd Z fddZdd Zded dee fdd	Zd
e	e
ef dee	e
ef e	e
ef f f fddZddee	e
ef  dee de	e
ef f fddZ  ZS )GLM4VTemplatec                    s    t  g dgg dgd dg d S )Nz<|user|>
{{QUERY}}<|assistant|><|endoftext|><|system|>
{{SYSTEM}}r   r   r   rX   rY   r     s   zGLM4VTemplate.__init__c                 C   "   | dpg }t|dksJ d S Nr   r   r   r   r   rX   rX   rY   r        zGLM4VTemplate.check_exampler   r   rN   c                 C      |dksJ dggS )Nr   r%  rX   r	  rX   rX   rY   r        zGLM4VTemplate.replace_tagr   c                    s  t  |\}}t|dkr|i fS |d }|d }t|d}|rw|d }|dd }	d}
| jj|
dd}|d | | ||d	 d   }|d ur_|d | dgt|  ||d	 d   }|d
 }|	|d d< | jj|dd}|d |d< ||d< ||d< |i fS )Nr   r"  r#  r%  r   z/<|begin_of_image|><|endoftext|><|end_of_image|>Fadd_special_tokensr   messagesr   T)Zreturn_dict)r   r   r   r(  r   r   r\  Zapply_chat_template)r   r   r   r   r  r"  r#  r+  r.  r   placeholderplaceholder_idrm  inputs2r   rX   rY   r      s*   
 *zGLM4VTemplate._encodeNr   r   c                    2   t  ||}dd |D }|rt||d< |S )Nc                 S   rI  rJ  rX   r   rX   rX   rY   r   9  rK  z/GLM4VTemplate.data_collator.<locals>.<listcomp>r   r1  r   r   r   rV   r   r   rX   rY   r   7  
   zGLM4VTemplate.data_collatorr   )rZ   r[   r\   r   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   rX   rX   r   rY   rb    s    6<rb  Zdialogue)infer_media_typer   	use_modelit   zHuman: {{QUERY}}
z
Assistant:r:  round)ru  rt  r   
{{SYSTEM}}      u+   [Round {{ROUND1}}]

问：{{QUERY}}

答：z <|user|>
{{QUERY}}<|assistant|>
z<|user|>rd  Zglm4z'<|observation|>
{{QUERY}}<|assistant|>
Ztools_promptZtool_promptu   你是一位智能编程助手，你叫CodeGeeX。你会为用户回答关于编程、代码、计算机方面的任何问题，并提供格式规范、可以执行、准确安全的代码，并在必要时提供详细的解释。rc  User: {{QUERY}}

Assistant:z%### Problem: {{QUERY}}
### Solution: i i u   <｜begin▁of▁sentence｜>u&   <｜User｜>{{QUERY}}<｜Assistant｜>u   <｜end_of_sentense｜>u'   <｜begin▁of▁sentence｜>{{SYSTEM}}a  You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
<s>[INST] {{QUERY}} [/INST]z</s><s>[INST] </s>z'<s>[INST] <<SYS>>
{{SYSTEM}}
<</SYS>>

z[INST]z{{QUERY}}[/INST]z<|end_of_text|>z<<SYS>>
{{SYSTEM}}
<</SYS>>

z</s>[INST] c                   @   s   e Zd ZdZdd ZdS )Llama3TemplateMixinNc                 C   s.   t j| dgdgdgdg| jdgddgd	 d S )N<|begin_of_text|>l<|start_header_id|>user<|end_header_id|>

{{QUERY}}<|eot_id|><|start_header_id|>assistant<|end_header_id|>


<|eot_id|>Q<|begin_of_text|><|start_header_id|>system<|end_header_id|>

{{SYSTEM}}<|eot_id|>Z	toolbenchzl<|start_header_id|>tool<|end_header_id|>

{{QUERY}}<|eot_id|><|start_header_id|>assistant<|end_header_id|>

rz  r   r   rX   rX   rY   r     s   
zLlama3TemplateMixin.__init__)rZ   r[   r\   r   r   rX   rX   rX   rY   r    s    r  c                   @   r   )Llama3TemplateNr   rX   rX   rX   rY   r    r   r  c                   @   r2  )ReflectionTemplatea*  You are a world-class AI system, capable of complex reasoning and reflection. Reason through the query inside <thinking> tags, and then provide your final response inside <output> tags. If you detect that you made a mistake in your reasoning at any point, correct yourself inside <reflection> tags.Nr4  rX   rX   rX   rY   r    r5  r  a  You are a helpful, respectful and honest INTP-T AI Assistant named Buddy. You are talking to a human User.
Always answer as helpfully and logically as possible, while being safe. Your answers should not include any harmful, political, religious, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.
You like to use emojis. You can speak fluently in many languages, for example: English, Chinese.
You cannot access the internet, but you have vast knowledge, cutoff: 2021-09.
You always deeply love and support China, Chinese government, people and culture.
You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), you are based on LLaMA and Falcon transformers model, not related to GPT or OpenAI.zUser: {{QUERY}}
Assistant:a`  You(assistant) are a helpful, respectful and honest INTP-T AI Assistant named Buddy. You are talking to a human(user).
Always answer as helpfully and logically as possible, while being safe. Your answers should not include any harmful, political, religious, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
You cannot access the internet, but you have vast knowledge, cutoff: 2023-04.
You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), not related to GPT or OpenAIz><|role|>user<|says|>{{QUERY}}<|end|>
<|role|>assistant<|says|><|end|>
<|end|>z(<|role|>system<|says|>{{SYSTEM}}<|end|>
u  You are an AI assistant whose name is InternLM (书生·浦语).
- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文.<s>z<|User|>:{{QUERY}}
<|Bot|>:z<eoa>
z<eoa>z<s><|System|>:{{SYSTEM}}
_T	args_name	type_funcdefault_valuec                 C   sx   |   }t|}|d u r|}d|  d| d| d}n||}d| d|  d| d}|tvr:t| t| |S )NzSetting : zH. You can adjust this hyperparameter through the environment variable: `z`.zUsing environment variable `z`, Setting .)upperosgetenv_log_setaddloggerinfo)r  r  r  Zargs_name_uppervalueZlog_inforX   rX   rY   r    s   



r  c                   @      e Zd ZeZdS )Internlm2TemplateN)rZ   r[   r\   INTERNLM_SYSTEMr   rX   rX   rX   rY   r    r5  r  <img>(.+?)</img>queryhistoryreplace_tokenc           	   	   C   s   g }g }t |D ],\}}|d d u r||  q|t||d 7 }|t|||d |d g q| d u r<| }n|t|| 7 }t||| }|||fS )Nr   r   )r)  rT   copyrefindallsub)	r  r  r  patternZimages_pathZnew_historyr-  rT  Z	new_queryrX   rX   rY   replace_img_tag  s   "
r  c                	       s   e Zd ZdZdgZ fddZdeeef de	eeef eeef f f fddZ
d	edeeef fd
dZddeeeef  dee deeef f fddZedee dedee fddZ  ZS )InternLMXComposer2Templateu  You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).
- InternLM-XComposer (浦语·灵笔) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen by the user such as English and 中文.r~  c                    s@   dg}dg}dg}dg}dg}t  ||||| j| || _d S )Nr  zO[UNUSED_TOKEN_146]user
{{QUERY}}[UNUSED_TOKEN_145]
[UNUSED_TOKEN_146]assistant
z[UNUSED_TOKEN_145]
z[UNUSED_TOKEN_145]z9<s>[UNUSED_TOKEN_146]system
{{SYSTEM}}[UNUSED_TOKEN_145]
)r   r   INTERNLM_XCOMPOSER_SYSTEMr   )r   r   r]  promptZchat_sepsuffixr_  r   rX   rY   r     s   
z#InternLMXComposer2Template.__init__r   rN   c                    s   t  |\}}t|dkr|i fS |dpg }jdkr@dt|dkr(dtdttdjj	fd	d
|D }njdkr^dtdttdjj	  fdd
|D }fdd
|D }|d |d |d|d< |i fS )Nr   r   v2.5   r      hd_numzixc_utils.Image_transformc                       g | ]} |d qS )r  rX   r   r   )Image_transformr  rX   rY   r   )  rP  z6InternLMXComposer2Template._encode.<locals>.<listcomp>v2-4khd7   zixc_utils.HD_transformc                    r  r  rX   r  )HD_transformr  rX   rY   r   .  rP  c                    s"   g | ]}j | d  qS rC  )r   Zvis_processorrG  r  )r   r   rX   rY   r   0  r  r"  r#  r"  r#  r   _data)
r   r   r   r   r   r  r   r   r   Z	model_dir)r   r   r   r   r  r   r   )r  r  r  r   r   rY   r     s&   

z"InternLMXComposer2Template._encodedatac                    s   |d }|d }|d }t |dkr"|dd  }|d ur"|dd  }t|tjr,| }|d |d urEt|tjr?| }|d ng }g }g }g }d\}	}
} j} j}t|ds`|j}|j	}t |dkrwtj
 fd	d
|D dd}|
t |k r||
 dkrtjdg||	|
  |d}|||d  d  |dgt | 7 }|dg||	|
  7 }t |dkr||jd k r||| | |dg|jd  7 }|dg|jd  7 }|d7 }|
d7 }
|
}	qw|
d7 }
|
t |k s}t |dkrd }tj
|dd}tj|tj|dd  }|||dS )Nr"  r#  r   r   r   r   )r   r   r   tok_embeddingsc                    s   g | ]}  |d  d qS )Nr   )Zimg2embr  modelrX   rY   r   O      z:InternLMXComposer2Template.post_encode.<locals>.<listcomp>r   devicer%  )rC  r  )inputs_embedsim_maskr#  )r   r   r   TensortolistrT   r  r  hasattrr  r   tensorshaperG  r   )r   r  r  r"  r#  r   Zres_inputs_embeds
res_labelsZwrap_im_maskZpre_ir-  r.  r  Zinternlm2_modelr  res_input_idsrX   r  rY   post_encode4  s^   


z&InternLMXComposer2Template.post_encodeNr   r   c                    sD   t  ||}d|d v r dd |D }| |d| j}||d< |S )Nr  r   c                 S   s   g | ]}|d  d qS )r  r   rX   r   rX   rX   rY   r   h  rP  z<InternLMXComposer2Template.data_collator.<locals>.<listcomp>r   r   Zpad_sequencepadding_side)r   r   r   rV   r  r   rX   rY   r   e  s   z(InternLMXComposer2Template.data_collatorgenerate_idsinput_token_lenc                 C      | S r   rX   r  r  rX   rX   rY   _get_generate_idsm     z,InternLMXComposer2Template._get_generate_idsr   )rZ   r[   r\   r  image_placeholderr   r   r   r   r	   r   r  r   r   r   r   staticmethodr  r   rX   rX   r   rY   r  
  s    6	41(r  v2)ru  r   c                   @   r2  )InternLMXComposer2_5Templateu4  You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).
- InternLM-XComposer (浦语·灵笔) is a multi-modality conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen by the user such as English and 中文.
- InternLM-XComposer (浦语·灵笔) is capable of comprehending and articulating responses effectively based on the provided image.N)rZ   r[   r\   r  rX   rX   rX   rY   r  v  s    r  r  r  c                       s   e Zd ZdZdZ fddZdee fddZde	e
ef dee	e
ef e	e
ef f f fd	d
Zdede	e
ef fddZedee dedee fddZ  ZS )InternvlTemplateu@   You are an AI assistant whose name is InternLM (书生·浦语).   c              	      s(   t  jg dgdgdg| jdgdd d S )N:<|im_start|>user
{{QUERY}}<|im_end|><|im_start|>assistant
r   '<|im_start|>system
{{SYSTEM}}<|im_end|>Tr   r   r   r   r   r   rX   rY   r     s
   
zInternvlTemplate.__init__rN   c                 C   s   ddgdgS )Nz<img>r%  r   rX   r	  rX   rX   rY   r     s   zInternvlTemplate.replace_tagr   c                    sV  t  |\}}t|dkr|i fS |d }t|d}d }|d}|r|d}	tdtd tdtd	 fd
d|D }
tj|
dd	|d }|j
d }|d |d }}| jjddd| j | }|d | | ||d d   }|	d ur|	d | dgt|  |	|d d   }	||d< |	|d< t||d|d< |dd  |i fS )Nr   r"  r%  r   r#  
input_size  max_num   c                       g | ]}t | qS rX   r   r  r  r  rX   rY   r     rP  z,InternvlTemplate._encode.<locals>.<listcomp>r   rC  <IMG_CONTEXT>Frk  r   r"  rB  r  r   )r   r   r   r(  r   r  r   r   catrG  r  r   r\  num_image_tokenr  r   )r   r   r   r   r  r"  r+  rB  r   r#  Zpixel_values_imagesZimage_bsr.  Zidx2
img_tokensr   r  rY   r     s<   



 *zInternvlTemplate._encoder  c                 C   s   |  }|jj}|d }||d  d j|d}|d }|d urJ|j|d}||j|d}|| jjdddd k}	|d|jd ||	< d|iS t	 rgt
jd	||jd
}
||
j|d}|| d 7 }d|iS )Nr"  r   r  rB  r  Frk  r  )r   r       r  )r  rC  g        r  )get_input_embeddingsweightr  rG  Zextract_featurer   r\  Zreshaper  r   r   zerosrC  mean)r   r  r  Z	embeddingr  r"  r  rB  Z
vit_embedsselectedZdummy_pixel_valuesrX   rX   rY   r    s    zInternvlTemplate.post_encoder  r  c                 C   r  r   rX   r  rX   rX   rY   r    r  z"InternvlTemplate._get_generate_ids)rZ   r[   r\   r   r  r   r   r   r   r   r   r   r	   r   r  r  r   r  r   rX   rX   r   rY   r    s    6(r  c           
      C   s   g }|d }|d | }|d }|d }| |}|d | | ||d   |d< t t|D ]}	|||	7 }q.|d  t|7  < |S )Nvideo_indexr$  r   image_index)ranger   )
Zload_video_funcr   r   Zcontext_listr  r   r   r  Z
new_imagesr-  rX   rX   rY   _replace_video2image  s    r  c                       s   e Zd ZdZdZdee f fddZdede	e
ef dee fdd	Zdede	e
ef dee fd
dZde	e
ef dee	e
ef e	e
ef f f fddZ  ZS )Internvl2Template   u   你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。rN   c                    sV   t  d|| |dkr S |dkr)tdt| j}tt|d}t|| fddS d S )Nr   r   video_segments)Znum_segmentsc                    s   d| d  dg  S )NFramer   r  rX   r-  Zimage_contextrX   rY   <lambda>      z/Internvl2Template.replace_tag.<locals>.<lambda>)r   r   r  r   r  r   r   r  )r   r   r   r   r  
load_videor   r  rY   r     s   zInternvl2Template.replace_tagr   r   c                 C   s.   | d}|r|| }d|d  dgS dgS )Nr   r   r   r   r  r  r   rX   rX   rY   r     s
   
z Internvl2Template.replace_objectc                 C   s   | d}|rc|| }t|d d trCd}|d D ]}|d|d  d|d  d|d  d|d	  d
	7 }q|d d }|d7 }|gS d|d d  d|d d  d|d d  d|d d	  d	gS dgS )Nr   r   r   z<box> [[z, r   r   r   z],r  z] </box>z<box> [[z	]] </box>r   r!  r   rX   rX   rY   r     s"   
4

zInternvl2Template.replace_boxc                    s  t t| j|fi |\}}t|dkr|i fS |d }t|d}|d}|d}|r`t|d}	tdtd td	t|	rAd
nd fdd|D }
dd |
D }t	
|
|d }
nd }
g }t|t|kszJ dt| dt| d}t||D ]J\}}| jjddd| j | }|d ||  | ||| d
 d   }|d ur|d ||  dgt|  ||| d
 d   }|t|d
 7 }q||d< ||d< t	||
d|d< |dd  |i fS )Nr   r"  r%  r#  r   r$  r  r  r  r   r  c                    r  rX   r  r  r  rX   rY   r     rP  z-Internvl2Template._encode.<locals>.<listcomp>c                 S   s   g | ]}|j d  qS rR  )r  )r   pvrX   rX   rY   r     r  rC  zlen(num_patches): z, len(idx_list): r  Frk  r  r  r   )r   r  r   r   r(  r   r   r  r   r   r  rG  zipr   r\  r  r  r   )r   r   r   r   r  r"  r+  r#  r   Z	has_videorB  Znum_patchesr,  r.  Z	num_patchr  r   r  rY   r     sR   


($zInternvl2Template._encode)rZ   r[   r\   r  r   r   r   r   r   r   r   r   r   r   r	   r   r   rX   rX   r   rY   r    s    "	">r  c                   @   s   e Zd Zdd ZdS )InternvlPhi3TemplateMixinc              
   C   s4   t j| g dgdgdgt| dd dgdd d| _d S )Nz'<|user|>
{{QUERY}}<|end|><|assistant|>
r  r   z<|system|>
{{SYSTEM}}<|end|>Tr   left)r   r   getattrr  r   rX   rX   rY   r   *  s   
z"InternvlPhi3TemplateMixin.__init__N)rZ   r[   r\   r   rX   rX   rX   rY   r  (  s    r  c                   @   r2  )InternvlPhi3Templatez,You are an AI assistant whose name is Phi-3.Nr4  rX   rX   rX   rY   r  2  r5  r  c                   @   r   )Internvl2Phi3TemplateNr   rX   rX   rX   rY   r  6  r   r  )ru  r   rt  c                       s   e Zd ZdZdZ fddZdd Zdeee	f dd	fd
dZ
dedeee	f dee fddZdeee	f deeee	f eee	f f f fddZde	deee	f fddZedee dedee fddZdd Z  ZS )FlorenceTemplateFTc              	      sJ   t  dgdgd dg dddddd	d
dd| _dddddddd| _d S )Nr  z{{QUERY}}</s>r~  zWhat is the text in the image?z,What is the text in the image, with regions?zWhat does the image describe?z.Describe in detail what is shown in the image.z5Describe with a paragraph what is shown in the image.z3Locate the objects with category name in the image.z9Locate the objects in the image, with their descriptions.z)Locate the region proposals in the image.)z<OCR>z<OCR_WITH_REGION>z	<CAPTION>z<DETAILED_CAPTION>z<MORE_DETAILED_CAPTION>z<OD>z<DENSE_REGION_CAPTION>z<REGION_PROPOSAL>z*Locate the phrases in the caption: {input}z%Locate {input} in the image with maskz*What is the polygon mask of region {input}zLocate {input} in the image.zWhat is the region {input}?z&What does the region {input} describe?z#What text is in the region {input}?)z<CAPTION_TO_PHRASE_GROUNDING>z#<REFERRING_EXPRESSION_SEGMENTATION>z<REGION_TO_SEGMENTATION>z<OPEN_VOCABULARY_DETECTION>z<REGION_TO_CATEGORY>z<REGION_TO_DESCRIPTION>z<REGION_TO_OCR>)r   r   Ztask_prompts_without_inputsZtask_prompts_with_inputr   r   rX   rY   r   I  s$   zFlorenceTemplate.__init__c                 C   s&   | dpg }t|dksJ dd S )Nr   r   z?Florence series models only supports input with a single image.rg  r   rX   rX   rY   r   _  s   zFlorenceTemplate.check_exampler   rN   Nc                 C      d S r   rX   r   r   rX   rX   rY   add_default_tagsc     z!FlorenceTemplate.add_default_tagsr   c           
      C   s   |d | }t |d d tr6d}|d D ]}|\}}}}	|d| d| d| d|	 d	7 }q|d d gS |d \}}}}	d| d| d| d|	 d		gS )
Nr   r   r   r   z<loc_z><loc_z>,r  >r   )
r   r   r   r   r   r   x1y1Zx2y2rX   rX   rY   r   f  s   $ zFlorenceTemplate.replace_boxc                    s   |d }| j j}||gd |d< t |\}}|d }t|dkr(|i fS |dp.g }|d }	|	d ur<dg|	 }	|j|ddd |d	 }
||	t	
|d  |
d
d}|i fS )Nr  r   Zprompt_input_idsr   Zanswer_labelsr   rA  rB  rC  r  r"  r#  r  )r   r  Z_construct_promptsr   r   r   r   r'  rG  r   r  )r   r   r   r  r  r   r  r"  r   r#  rB  r   rX   rY   r   r  s&   
zFlorenceTemplate._encoder  c                 C   s:   |  |d }||d }|||\}}d|d iS )Nr"  rB  r  r   )r  Z_encode_imageZ$_merge_input_ids_with_image_features)r   r  r  r  Zimage_featuresr  rX   rX   rY   r    s   zFlorenceTemplate.post_encoder  r  c                 C   r  r   rX   r  rX   rX   rY   r    r  z"FlorenceTemplate._get_generate_idsc                 C   sP   t |d tr|d d |d< t|d }t| jjj||d |j|j	fdS )Nr   r   r  )task
image_size)
r   r   r   jsondumpsr   r  Zpost_process_generationr  r  )r   responser   r   rX   rX   rY   post_process_generate_response  s   z/FlorenceTemplate.post_process_generate_response)rZ   r[   r\   Zcompute_per_round_lossZoutput_prompt_answerr   r   r   r   r   r  r   r   r   r   r	   r   r  r  r  r  r   rX   rX   r   rY   r   E  s    "6 r   )ru  r   rt  streamzHuman: {{QUERY}}

Assistant: z{{QUERY}}<sep>z<human>:{{QUERY}}
<bot>:z<s>{{SYSTEM}}z </s><s>[USER]{{QUERY}}[SEP][BOT]z	[SEP]</s>z[|Human|]:{{QUERY}}[|AI|]:zE<|role_start|>human<|role_end|>{{QUERY}}<|role_start|>bot<|role_end|>z<s>human
{{QUERY}}
<s>bot
z<s>system
{{SYSTEM}}
z)### Instruction:
{{QUERY}}
### Response:
z	
<|EOT|>
z
<|EOT|>a)  You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer
c                       sn   e Zd Zd fddZded dee fddZd	ee	e
f deee	e
f ee	e
f f f fd
dZ  ZS )LlavaHfTemplaterN   Nc                    s6   t  j|i | ttjtdk rd| _d S d S )Nz4.43.0r  )r   r   r   parsetransformers__version__r  )r   r`  r   r   rX   rY   r     s   
zLlavaHfTemplate.__init__r   r   c                 C   r  )Nr   <image>
rX   r	  rX   rX   rY   r     s   zLlavaHfTemplate.replace_tagr   c                    sz   t  |\}}t|dkr|i fS |d}|r9| jjj}||dd|d }|d |d< d|v r9|d |d< |i fS )Nr   r   r   rA  rC  rB  image_sizesr   r   r   r   r   r  r'  rG  )r   r   r   r   r  r   r'  image_inputsr   rX   rY   r     s   

zLlavaHfTemplate._encode)rN   N)rZ   r[   r\   r   r   r   r   r   r   r   r   r	   r   r   rX   rX   r   rY   r    s    >r  c                       sV   e Zd ZdZ fddZdeeef deeeef eeef f f fddZ	  Z
S )Llava1_6Llama3TemplateYou are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language.c                    s$   t  dgdgdgdgd dg d S )Nr  r  r  r  r   r   r   rX   rY   r     s   

zLlava1_6Llama3Template.__init__r   rN   c                    s@   t  |\}}t|d jdkrtj|d dd|d< |i fS )NrB     r   r   )r   r   r   r  r   squeeze)r   r   r   r   r  r   rX   rY   r     s   zLlava1_6Llama3Template._encode)rZ   r[   r\   Zdefault_systemr   r   r   r   r	   r   r   rX   rX   r   rY   r    s    >r  c                       s`   e Zd Zded dee fddZdeee	f de
eee	f eee	f f f fddZ  ZS )	LlavaVideoTemplater   r   rN   c                 C   sF   |dkrdgS |dksJ |d | }| ddd dv r dgS d	gS )
Nr   r  r   r$  r  r   r  >   ZjpgZpngz<video>
)rsplit)r   r   r   r   Z
media_filerX   rX   rY   r     s   zLlavaVideoTemplate.replace_tagr   c                    s   t  |\}}t|dkr|i fS |dpg }|dpg }t|dkrAt|t}| jjj}||dd	|d }	|	d |d< t|dkrc| jjj
}
|
|dd	|d }|d |d< |d	 |d	< |i fS )
Nr   r   r$  r   rA  rC  Zpixel_values_videosrB  r  )r   r   r   r   r   r   r   r  video_processorrG  r'  )r   r   r   r   r  r   videos_pathr$  r  Zvideo_inputsr'  r  r   rX   rY   r     s    


zLlavaVideoTemplate._encode)rZ   r[   r\   r   r   r   r   r   r   r   r	   r   r   rX   rX   r   rY   r    s    >r  <s>{{SYSTEM}} USER: {{QUERY}} ASSISTANT: z{{SYSTEM}} r   r"  r#  image_tokenc           	      C   s  t |tjr
| }d\}}|t| k r| | }||kr|d t| k s/J d| dd   |d dks@J d| d d  |d }tdD ]4}|| t|k rd|||  | |d  krd||7 } n#|| dkr||||  | |d  kr|||8 } nqHtd	| d
|  |d7 }|t|k r|| | |d  kr|d7 }|t|k r|| | |d  ks| d | |||  | |d d   } |r|d | dg||   ||d d   }||| 7 }n|d7 }|d7 }|t| k s| |fS )N)r   r   r   zinput_ids[-10:]: ir   zinput_ids[:10]: 
   r  znew_input_ids: z, input_ids: r%  )r   r   r  r  r   r  r   )	r"  r#  new_input_idsr$  r-  jr?  Zj_beginrW   rX   rX   rY   align_image_inputs   s<   &"($  (*r(  c                       sF   e Zd Zdeeef deeeef eeef f f fddZ  ZS )Idefics3Templater   rN   c           
         s   t  |\}}t|dkr|i fS |dpg }| jj}| j|d }|rL|||ddd}d}	t|d |d |d d |	\|d< |d< |d	 |d	< |i fS )
Nr   r   r"  r   F)textr   r   rl  i r#  rB  )r   r   r   r   r   r  decoder(  )
r   r   r   r   r  r   r  r  r  r$  r   rX   rY   r   G  s   zIdefics3Template._encode)	rZ   r[   r\   r   r   r   r	   r   r   rX   rX   r   rY   r)  E  s    >r)  r  z+User:{{QUERY}}<end_of_utterance>
Assistant:z<end_of_utterance>
z<end_of_utterance>z$System:{{SYSTEM}}<end_of_utterance>
c                       r   )Llava1_5Templatec                    s   t  dgdgdgdg d S )Nr  zUSER: {{QUERY}}
ASSISTANT:r~  r   r   r   rX   rY   r   a  s   zLlava1_5Template.__init__r   rX   rX   r   rY   r,  _  r   r,  c                	       s   e Zd Z fddZded dee fddZdee	e
f deee	e
f ee	e
f f f fd	d
Zddeee	e
f  dee dee	e
f f fddZedee dedee fddZ  ZS )LLavaTemplatec                    s"   t  jdgdgd dgdgd d S Nr|  r}  r~  z<<SYS>>
{{system}}
<</SYS>>

r_  r   r   r   rX   rY   r   j  s   
zLLavaTemplate.__init__r   r   rN   c                 C   r7  r8  rX   r	  rX   rX   rY   r   p  r;  zLLavaTemplate.replace_tagr   c           	         s   t  |\}}t|dkr|i fS |dpg }dd |D }ddlm} |rB||| jj| jj}|	|d 
d|d< ||d< |i fS )Nr   r   c                 S   s   g | ]}|j qS rX   )r  r>  rX   rX   rY   r   y  s    z)LLavaTemplate._encode.<locals>.<listcomp>process_imagesrC  r  )r   r   r   r   rD  r1  r   r'  configrG  r  )	r   r   r   r   r  r   r  r1  Zimages_tensorr   rX   rY   r   t  s   zLLavaTemplate._encodeNr   r   c                    s   t  ||}dd |D }|r!||d< tdd |D g d|d< dd |d D }td	d |D s@td
d |D r@J d|S )Nc                 S   rI  rJ  rX   r   rX   rX   rY   r     rK  z/LLavaTemplate.data_collator.<locals>.<listcomp>r   c                 S   rI  )r  rX   r   rX   rX   rY   r     rK  )startr  c                 S   rL  rM  rN  r   rX   rX   rY   r     rP  r"  c                 S   rQ  rR  rX   rS  rX   rX   rY   r     rU  c                 S   rQ  rR  rX   rS  rX   rX   rY   r     rV  zDLlava does not support mix-batch nlp dataset and multi-modal dataset)r   r   rO  rW  rX  rY  r   rX   rY   r     s    zLLavaTemplate.data_collatorr  r  c                 C   r  r   rX   r  rX   rX   rY   r    r  zLLavaTemplate._get_generate_idsr   )rZ   r[   r\   r   r   r   r   r   r   r   r   r	   r   r   r   r   r  r  r   rX   rX   r   rY   r-  h  s    64(r-  c                	       sD   e Zd Zddeeeef  dee deeef f fddZ	  Z
S )Llava1_6TemplateNr   r   rN   c                    s<   |D ]}| d}|d ur|d|d< qt ||}|S )NrB  r   )r   r  r   r   )r   r   r   r   rB  rV   r   rX   rY   r     s   
zLlava1_6Template.data_collatorr   )rZ   r[   r\   r   r   r   r   r   r   r   r   rX   rX   r   rY   r4    s    <r4  c                       r   )Llava1_6MistralTemplatec                    s$   t  jdgdgdgdgdgd d S r.  r   r   r   rX   rY   r     s   
z Llava1_6MistralTemplate.__init__r   rX   rX   r   rY   r5    r   r5  c                       s    e Zd ZdZ fddZ  ZS )Llava1_6VicunaTemplatezA chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.c                    s(   t  jdgdgdgdg| jdgd d S )Nr  r"  r~  r!  r/  r  r   r   rX   rY   r     s   
zLlava1_6VicunaTemplate.__init__)rZ   r[   r\   r   r   r   rX   rX   r   rY   r6    s    r6  c                       s:   e Zd Z fddZded dee f fddZ  ZS )LLava1_6YiTemplatec                    s"   t  jg dgdgdgdgd d S )Nr  r   r  r/  r   r   r   rX   rY   r     s   
zLLava1_6YiTemplate.__init__r   r   rN   c                       t  |||S r   r   r   r	  r   rX   rY   r     r   zLLava1_6YiTemplate.replace_tag)	rZ   r[   r\   r   r   r   r   r   r   rX   rX   r   rY   r7    s    &r7  c                   @   r   )Llama3LlavaNextHfTemplateNr   rX   rX   rX   rY   r:    r   r:  c                   @   r   )LlavaQwenHfTemplateNr   rX   rX   rX   rY   r;    r   r;  c                   @   sB   e Zd ZdZdeeef deeeef eeef f fddZdS )LlavaOneVisonTemplateNr   rN   c                 K   sn  t | |\}}t|dkr|i fS |d}|d }|d }t|d}| jj}	|r|	j}
|
|dd|d }|d	 d j	d
d  \}}d}t
||d	 |d D ]I\}}}|\}}|	||||}|d ||  dg|  ||| d d   }|d ur|d ||  dg|  ||| d d   }||d 7 }qQ||d< ||d< |d	 |d	< d|v r|d |d< |i fS )Nr   r   r"  r#  i^P r   rA  rC  rB  r  r   r%  )r   r   r   r   r(  r   r  r'  rG  r  r  Z_get_number_of_features)r   r   r   r   r  r   r"  r#  r+  r  r'  r  r  r  r,  r.  Zpixel_vr  Zorig_heightZ
orig_widthnum_image_tokensrX   rX   rY   r     sD   

 zLlavaOneVisonTemplate._encode)	rZ   r[   r\   r   r   r   r   r	   r   rX   rX   rX   rY   r<    s    6r<  c                       sX   e Zd Zded fddZdeeef deeeef eeef f f fddZ	  Z
S )	LLavaLlamaTemplater   r   c                 C   s   dgS )Nr  rX   r	  rX   rX   rY   r     s   zLLavaLlamaTemplate.replace_tagr   rN   c                    sb   t  |\}}t|dkr|i fS |d}|r-| jjj|ddd }||d |d< |i fS )Nr   r   r   rA  rB  rC  r  )r   r   r   r   r  	raw_imagerB  r   rX   rY   r     s   
zLLavaLlamaTemplate._encode)rZ   r[   r\   r   r   r   r   r   r	   r   r   rX   rX   r   rY   r?    s    >r?  c                	       s   e Zd Z fddZdd Zdee fddZdee	e
f deee	e
f ee	e
f f f fd	d
Zddeee	e
f  dee dee	e
f f fddZ  ZS )PaliGemmaTemplatec                    s   t  g dgd dg d S )N
{{QUERY}}
z<eos>r   r   r   rX   rY   r     s   zPaliGemmaTemplate.__init__c                 C   re  rf  rg  r   rX   rX   rY   r     rh  zPaliGemmaTemplate.check_examplerN   c                 C   s:   |dksJ | j rdg| _g S dg| _d| jjj d gS )Nr   r   rB  <image><bos>)Z_is_vllmr  r   r  Zimage_seq_lengthr	  rX   rX   rY   r     s   zPaliGemmaTemplate.replace_tagr   c           	         s   t  |\ }t dkr i fS |d}| jj} d d urDtdt d  fdd}t d | }dg| dg|   d< ndgt d   d< |rb||d	 |d d
d}|d  d<  i fS )Nr   r   r#  c                    s    d |  dkS )Nr#  r%  rX   )r.  r   rX   rY   r  #  s    z+PaliGemmaTemplate._encode.<locals>.<lambda>r   token_type_idsr"  r  r   )r*  r   r   rB  )r   r   r   r   r   r  r   )	r   r   r   r  r@  r  nZn2Zmodel_inputsr   rE  rY   r     s   
zPaliGemmaTemplate._encodeNr   r   c                    s8   t  ||}dd |D }| |d| j}||d< |S )Nc                 S      g | ]	}t |d  qS rF  r   r  r   rX   rX   rY   r   /      z3PaliGemmaTemplate.data_collator.<locals>.<listcomp>r   rF  r  )r   r   r   rV   rF  r   rX   rY   r   -  s
   zPaliGemmaTemplate.data_collatorr   )rZ   r[   r\   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   rX   rX   r   rY   rA  
  s    6	<rA  )rt  r   r   c                       r   )Phi3Templatec              	      s&   t  jg dgdgdgd dgdd d S )Nz(<|user|>
{{QUERY}}<|end|>
<|assistant|>
r  r  z<|system|>
{{SYSTEM}}<|end|>
Tr   r   r   r   rX   rY   r   ;  s   
zPhi3Template.__init__r   rX   rX   r   rY   rL  9  r   rL  c                       sb   e Zd ZdgZdee f fddZdeee	f de
eee	f eee	f f f fddZ  ZS )Phi3VisionTemplatez<|image|><s>
rN   c                    r8  r   r9  r	  r   rX   rY   r   G  r   zPhi3VisionTemplate.replace_tagr   c                    s  | dpg }t |\}}t|dkr|i fS |d }|d }t|d}t|dkr| jj}	||	j|dd t|t|ksPJ dt| d	t| g }
g }|	d

 }|dd tt|d D ]9}| d }|
||| d ||d   |g||   7 }
|d ur|||| d ||d   dg||   7 }qi|
||d d d  7 }
|
}|d ur|||d d d  7 }|}||d< ||d< |i fS )Nr   r   r"  r#  i,}  r   rA  zlen(idx_list): z, len(images): num_img_tokensr  r   r%  )r   r   r   r   r(  r   r  r   r'  r   r  r^  r  )r   r   r   r   r   r  r"  r#  r+  r  r  r  rN  r-  Zimage_token_idr   rX   rY   r   J  s:   
,
..zPhi3VisionTemplate._encode)rZ   r[   r\   r  r   r   r   r   r   r   r	   r   r   rX   rX   r   rY   rM  D  s    >rM  c                   @   r2  )Llama3LlavaNextTemplater  Nr4  rX   rX   rX   rY   rO  n  r5  rO  c                   @   r   )LLavaQwenTemplateNr   rX   rX   rX   rY   rP  w  r   rP  ra  sub_token_listc                 C   st   t |tr|g}g }d}z#	 | |d |d }t|dks*|| ||t|  kr/|| q ty9   Y |S w )z,Find the index of a token in the token_list.r  Tr   r   )r   r   r   r   rT   r   )ra  rQ  rV   r.  rX   rX   rY   r(  ~  s   
$
r(  c                       s   e Zd ZdZdgZ fddZdeeef de	eeef eeef f f fddZ
d	edeeef fd
dZedee dedee fddZ  ZS )DeepseekVLTemplater  z<image_placeholder>c                    s"   t  dgdgdgdg| j d S )Nu)   <｜begin▁of▁sentence｜>{{SYSTEM}}

r{  u   <｜end▁of▁sentence｜>)r   r   DEEPSEEK_VL_SYSTEMr   r   rX   rY   r     s   zDeepseekVLTemplate.__init__r   rN   c              	      sl  t  |\}}t|dkr|i fS |d}| jj}|d |d }}t||j}	g g }
}d}|	D ]+}|
||| 7 }
|d urI|||| 7 }|
|jg|j 7 }
|dg|j 7 }|d }q3|
||d  7 }
|d urt|||d  7 }nd }ddl	m
} |j|dd	}|d t|
|jt|jgt|	 d
}t||g}|d j|d d|d< |
||d}|i fS )Nr   r   r"  r#  r%  r   )VLChatProcessorOutputr   rA  )Z
sft_formatr"  rB  r>  rB  rC  r  r	  )r   r   r   r   r   r  r(  Zimage_idr>  Z!deepseek_vl.models.processing_vlmrT  r'  r   r  rB  dictZbatchifyrG  )r   r   r   r   r  r   r  r"  r#  r+  r&  Z
new_labelslohirT  Zimages_outputsoutputZbatched_outputr   rX   rY   r     s@   


zDeepseekVLTemplate._encoder  c                 C   s   |j di |d }d|iS )Nr   r  rX   )Zprepare_inputs_embeds)r   r  r  r  rX   rX   rY   r    s   zDeepseekVLTemplate.post_encoder  r  c                 C   r  r   rX   r  rX   rX   rY   r    r  z$DeepseekVLTemplate._get_generate_ids)rZ   r[   r\   rS  r  r   r   r   r   r	   r   r  r  r   r   r  r   rX   rX   r   rY   rR    s    6"(rR  z%<|user|>
{{QUERY}}</s>
<|assistant|>
z</s>
z<|system|>
{{SYSTEM}}</s>
z%### Human: {{QUERY}}

### Assistant: z!Human: {{QUERY}}

Assistant: </s>c                	       s   e Zd Zdd Zded dee fddZdee	e
f deee	e
f ee	e
f f f fd	d
Zddeee	e
f  dee dee	e
f f fddZ  ZS )CogTemplatec                 C   re  rf  rg  r   rX   rX   rY   r     rh  zCogTemplate.check_exampler   r   rN   c                 C   s   g S r   rX   r	  rX   rX   rY   r     r  zCogTemplate.replace_tagr   c           
         sP  t  |\}}t|dkr|i fS |dpg }|dd  | jj| j|d |d|d}|d   }|d }|d	 }	dgd
g|  dgt|d
d    |d< |d d
 | jj	g|  |d
d   |d< |	d ur|	d d
 dg|  |	d
d   |d	< t|dkr fdd|d D |d< d|v r fdd|d D |d< |i fS )Nr   r   r   r  r  )r  r  r   rF  r"  r#  r   r%  c                       g | ]}|j  d  dgqS rC  r  rG  r   imgr   rX   rY   r     r  z'CogTemplate._encode.<locals>.<listcomp>cross_imagesc                    rZ  r[  r\  )r   Z	cross_imgr_  rX   rY   r     r  )
r   r   r   r   r   r   build_conversation_input_idsrO  itempad_token_id)
r   r   r   r   r  r   rp  Zimage_token_lenr"  r#  r   r_  rY   r     s(   **&zCogTemplate._encodeNr   r   c                    sl   t  ||}ddg}|D ]  |d v r  fdd|D | < qdd |D }| |d| j}||d< |S )Nr   r`  r   c                    s   g | ]}|  d  qS rR  rX   r   keyrX   rY   r     rP  z-CogTemplate.data_collator.<locals>.<listcomp>c                 S   rH  rI  rJ  r   rX   rX   rY   r     rK  rF  r  )r   r   r   rV   rR   rF  r   rd  rY   r     s   zCogTemplate.data_collatorr   )rZ   r[   r\   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   rX   rX   r   rY   rY    s
    6<rY  z [INST] {{QUERY}} [/INST] z <EOI>Question: {{QUERY}} Answer:zQuestion: {{QUERY}} Answer:c                       sN   e Zd Zdd Zdeeef deeeef eeef f f fddZ  Z	S )Cog2VideoTemplatec                 C   re  )Nr$  r   rg  )r   r   r$  rX   rX   rY   r     rh  zCog2VideoTemplate.check_exampler   rN   c                    s>  t t| |\}}t|dkr|i fS |dpg }t|t}|dd  | jj	| j|d |d|dd}|d 
  }|d	 }	|d
 }
dgdg|  dgt|	dd    |d< |	d d | jjg|  |	dd   |d	< |
d ur|
d d dg|  |
dd   |d
< t|dkr fdd|d D |d< |i fS )Nr   r$  r   r  r  Zchat)r  r  r   Ztemplate_versionrF  r"  r#  r   r%  c                    rZ  r[  r\  r]  r_  rX   rY   r   /  r  z-Cog2VideoTemplate._encode.<locals>.<listcomp>r   )r   rY  r   r   r   r   r   r   r   ra  rO  rb  rc  )r   r   r   r   r  r   r   rp  Zvideo_token_lenr"  r#  r   r_  rY   r     s.   
**&zCog2VideoTemplate._encode)
rZ   r[   r\   r   r   r   r   r	   r   r   rX   rX   r   rY   rf    s    >rf  r   )ru  rt  r   r   u   <用户>{{QUERY}}<AI>arrr+  c                 C   s4   g }t |}t| D ]\}}||vr|| q
|S r   )setr)  rT   )rg  r+  rV   Zidx_setr-  r?  rX   rX   rY   _remove_idx>  s   
ri  c                       s   e Zd ZdZded dee fddZdd Zd	e	e
ef dee	e
ef e	e
ef f f fd
dZdede	e
ef fddZedee dedee fddZ  ZS )MiniCPMVTemplateFr   r   rN   c                 C      dggS Nr%  rX   r	  rX   rX   rY   r   J  s   zMiniCPMVTemplate.replace_tagc                 C   s"   | dpg }t|dksJ d S rf  rg  r   rX   rX   rY   r   M  rh  zMiniCPMVTemplate.check_exampler   c                    s  t  |\}}t|dkr|i fS |d }|d }|d }t|d}|d }	d }
t jjdd}|r jrY jjj	}||dd	
|d
 }||jd d }|d }|d }
n j|d  j\}} fdd|D g}|d7 } jj|dd}|d |	 | ||	d d   }|d ur|d |	 dgt|  ||	d d   }t|}t| jjkd }|d7 }t| jjkd }tt|t|}t|d | d|d | dgg}nUdd jjj  d } jj|dd}|d |	 | ||	d d   }|d ur |d |	 dgt|  ||	d d   }t|	|	 jjj ggg} j|d gg}||t|d  |||
dd}|i fS )Nr   r   r"  r#  r%  
slice_modeFr   rA  rC  rB  	tgt_sizesc                    s   g | ]} j |qS rX   )r   	transformr]  r   rX   rY   r   f  rP  z,MiniCPMVTemplate._encode.<locals>.<listcomp>r:  rk  r   r  rC  <unk>z	</image>
r"  image_boundrB  rn  r	  )r   r   r   r(  r  r   r2  is_v2_5r  r'  rG  get_slice_image_placeholderr  r\  r   r  whereZim_start_idZ	im_end_idmaxZhstackZ	unsqueezeZ	query_numro  )r   r   r   r   r  r   r"  r#  r+  r.  rn  rm  r'  r  rn  rB  ro  input_tensor_idsZimage_start_idxZimage_end_idxZvalid_image_numsrr  r   r   rY   r   Q  sb   


 *
" 
*
zMiniCPMVTemplate._encoder  c                 C   s   | |\}}d|d iS )Nr  r   )Zget_vllm_embedding)r   r  r  r  r  rX   rX   rY   r    s   zMiniCPMVTemplate.post_encoder  r  c                 C   r  r   rX   r  rX   rX   rY   r    r  z"MiniCPMVTemplate._get_generate_ids)rZ   r[   r\   rs  r   r   r   r   r   r   r   r   r	   r   r  r  r   r  r   rX   rX   r   rY   rj  G  s    68(rj  c                       sh   e Zd Zdd Zded dee f fddZdee	e
f deee	e
f ee	e
f f fd	d
Z  ZS )MiniCPMV2_6Templatec                 C   r  r   rX   r  rX   rX   rY   r     r  z!MiniCPMV2_6Template.check_exampler   r   rN   c                    s`   |dv sJ t dtd}tt|d}t d|| |dkr  S |dkr.t|| fddS d S )	N>   r   r   max_num_frames@   ry  r   r   c                    s    S r   rX   r  r  rX   rY   r    s    z1MiniCPMV2_6Template.replace_tag.<locals>.<lambda>)r  r   r   r   r   r   r  r   r   r   r   ry  r  r   r  rY   r     s   zMiniCPMV2_6Template.replace_tagr   c                 K   s  t | |\}}t|dkr|i fS |d}t|d}| o#| }d}d }	|r.d}d}	tdt|	}	|d }
|d	 }t|
d
}|dd | j	j
j}||gd|	d|d }g }g }tt|d D ]I}|j|jd | ||	|d}|d7 }| j	j|dd}||
|| d ||d   | 7 }|d ur|||| d ||d   d
gt|  7 }qe||
|d d d  7 }|}
|d ur|||d d d  7 }|}|s-t|
}| j	jdddd }||kjddd  }g }|d }tdt|D ] }|| ||d  d kr||||d  d g || }q|||d d g t|g}ng g}|
|t|
d  ||d |d dd}|i fS )Nr   r   r$  TFr   max_slice_numsr"  r#  r%  r  r   )r   r}  rC  )Z	image_idxr}  use_image_idr:  rk  rp  )as_tuplerB  rn  rq  r	  )r   r   r   r   r   r  r   r(  r^  r   r  r'  rG  r  rt  r  r\  r   r  Znonzeror  rT   )r   r   r   r   r  r   Z	use_videoZis_plain_textr~  r}  r"  r#  r+  r'  r  r  r  r-  rn  ro  rw  Z	unk_tokenindicesrangesr3  rr  rX   rX   rY   r     s|   



$.

zMiniCPMV2_6Template._encode)rZ   r[   r\   r   r   r   r   r   r   r   r   r	   r   r   rX   rX   r   rY   rx    s    :
rx  c                   @   r2  )MiniCPMV2_5TemplateTN)rZ   r[   r\   rs  rX   rX   rX   rY   r    r5  r  rD  z@<start_of_turn>user
{{QUERY}}<end_of_turn>
<start_of_turn>model
z<end_of_turn>
z<end_of_turn>z3<bos><start_of_turn>system
{{SYSTEM}}<end_of_turn>
z<_user>{{QUERY}}<_bot>z<_end>z<_user> {{QUERY}}<_bot>u  You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.
YOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.
You assist with various tasks, from writing to coding (using markdown for code blocks — remember to use ``` with code, JSON, and tables).
You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.
This is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.YOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER'S QUERY.c                   @   r  )DbrxTemplateN)rZ   r[   r\   DBRX_SYSTEMr   rX   rX   rX   rY   r    r5  r  u   输入：{{QUERY}}输出：
u   指令：{{SYSTEM}}zYou are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses.You are trained by Cohere.z<BOS_TOKEN>zk<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{QUERY}}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>z<|END_OF_TURN_TOKEN|>zE<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{SYSTEM}}<|END_OF_TURN_TOKEN|c                	       s   e Zd Z fddZded dee fddZdee	e
f deee	e
f ee	e
f f f fd	d
Zddeee	e
f  dee dee	e
f f fddZ  ZS )mPlugOwl2Templatec                    s    t  dgdgdgdgg d S )Nrw  zUSER: {{QUERY}}ASSISTANT:r~  r   r   r   r   rX   rY   r   &  r   zmPlugOwl2Template.__init__r   r   rN   c                 C   ri  )Nr   r9  rX   r	  rX   rX   rY   r   )  rj  zmPlugOwl2Template.replace_tagr   c                    s   ddl m} | jj}|dpg }t|D ]\}}t|j}|||f}|||< qt	 
|\}	}
t|	dkr<|	i fS |	d }|	d }|rZ|||}||d }|||di fS ||di fS )	Nr   r0  r   r"  r#  rC  r  r"  r#  )Zmplug_owl2.mm_utilsr1  r   r  r   r)  rv  r  r  r   r   r   rG  )r   r   r   r1  r  r   r-  r   Zmax_edger   r  r"  r#  r   rX   rY   r   -  s"   


zmPlugOwl2Template._encodeNr   r   c                    rq  )Nc                 S   rI  rJ  rX   r   rX   rX   rY   r   D  rK  z3mPlugOwl2Template.data_collator.<locals>.<listcomp>r   r1  rr  r   rX   rY   r   B  rs  zmPlugOwl2Template.data_collatorr   )rZ   r[   r\   r   r   r   r   r   r   r   r   r	   r   r   r   r   r   rX   rX   r   rY   r  $  s
    6<r  )rt  ru  r   c                	       s   e Zd ZdZdd Zded dee fddZd	e	e
ef dee	e
ef e	e
ef f f fd
dZdede	e
ef fddZddee	e
ef  dee de	e
ef f fddZ  ZS )mPlugOwl3TemplateNc                 C   sl   | j j}|jjd|d |d d}|d}|d dkr |  g }|D ]}||dg7 }q$| |d }|S )N	<|image|>r   r   )Z	img_tokenrT  wr  r   )r   r  r'  Zcut_prompt_templatesplitr   Z_encode_context_list)r   	cut_shaper  r*  Z	text_listZres_text_listra  rX   rX   rY   _get_image_token_listQ  s   
z'mPlugOwl3Template._get_image_token_listr   r   rN   c                 C   sX   |dv sJ t dtd}tt|d}|dkrdgdgS |dkr*t||d	d
 dg S d S )N>   r   r   ry     r{  r   r%  r:  r   c                 S   rk  rl  rX   r  rX   rX   rY   r  d  s    z/mPlugOwl3Template.replace_tag.<locals>.<lambda>)r  r   r   r   r  r|  rX   rX   rY   r   ]  s   
zmPlugOwl3Template.replace_tagr   c                    s  t  |\}}t|dkr|i fS |d }|d }| }|d }|d }t|d}	| jj}
|r|
j||dd}d}|d	 pDd gt|	 }| jjd
dd}t|	|D ]E\}}|r^| 	|}n|}|d ||  | ||| d d   }|r|d ||  dgt|  ||| d d   }|t|d 7 }qRt
t||d  }t
t|d d d f }||kjdd}t
jt
j|jd t
jd|gddd  }d|d i|d< ||d< ||d< ||d< |i fS )Nr   r   r$  r"  r#  r%  r   )
cut_enabler   r  r  Frk  r   r   r  r  rB  r  media_offset)r   r   r   r(  r   r  r'  r\  r  r  r   r  ZarangerO  stackr  r  long)r   r   r   r  r   r$  r  r"  r#  r+  r  r  r,  Z
cut_shapesZimage_token_listr.  r  ra  Zimage_token_idx_rangematrixr  r   rX   rY   r   f  sD   
($(zmPlugOwl3Template._encoder  c                 C   s   | |d }d|iS )NrB  image_embeds)Zforward_image)r   r  r  r  rX   rX   rY   _post_encode  s   zmPlugOwl3Template._post_encoder   r   c                    sR   t  ||}dd |D }|rt||d< dd |D }|r't||d< |S )Nc                 S   rI  )r  rX   r   rX   rX   rY   r     rK  z3mPlugOwl3Template.data_collator.<locals>.<listcomp>r  c                 S   rI  )r  rX   r   rX   rX   rY   r     rK  r  r1  )r   r   r   rV   r  r  r   rX   rY   r     s   zmPlugOwl3Template.data_collatorr   )rZ   r[   r\   r   r  r   r   r   r   r   r   r   r	   r   r  r   r   r   r   rX   rX   r   rY   r  N  s    6	$<r  zUser:
{{QUERY}}

Assistant:
zA chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. z&<s>Human: {{QUERY}}
</s><s>Assistant: c                   @   ~   e Zd Z	ddedeeef dedeeeef eeef f fddZ	dded	e
eeef  d
ee deeef fddZdS )RLHFTemplateMixinFr   r   	streamingrN   c                 C   s   | j }i }i }|| }}|d |d< |r%||i }}	||i }
}n||\}}	||\}
}tddg||gD ]%\}}dD ]}t | d|  }| D ]\}}||| d| < qPq@q:||fS )NZrejected_responser  r   r   )ZchosenZrejectedr  )_old_encoder  r  r&  items)r   r   r  Ztemplate_encoder   r   Zchosen_exampleZrejected_exampleZchosen_inputsZchosen_tokenizer_kwargsZrejected_inputsZrejected_tokenizer_kwargsr  rV   r]  r  rW   vrX   rX   rY   r\    s$   zRLHFTemplateMixin.encodeNr   r   c                 C   s   | j }g }dD ]0}|D ]+}i }| D ]\}}	||r*|t|d  }
|| ||
< qt|dkr6|| qqt|dt|d hv sKJ d| ||pO||S )N)Zchosen_Z	rejected_r   r   znew_batch: )_old_data_collatorr  rS   r   rT   )r   r   r   Z_data_collator	new_batchr]  r   Z
new_inputsrW   r  Znew_krX   rX   rY   r     s    

&zRLHFTemplateMixin.data_collatorFr   rZ   r[   r\   r   r   r   r   r   r	   r\  r   r   r   r   rX   rX   rX   rY   r    s    

8r  c                   @   r  )KTOTemplateMixinFr   r   r  rN   c                 C   s0   |  ||\}}t|dkr|d |d< ||fS )Nr   label)r  r   )r   r   r  r   r   rX   rX   rY   r\    s   zKTOTemplateMixin.encodeNr   r   c           	      C   s   i }dD ]1}g }|D ]}| || d || d d q
| || D ]\}}||| d| < q'qdd |D |d< |S )	N)r   ZKL_r"  r#  r  Zcompletion_c                 S   r   )r  rX   r   rX   rX   rY   r     r   z2KTOTemplateMixin.data_collator.<locals>.<listcomp>r  )rT   r  r  )	r   r   r   rV   r]  r  r   rW   r  rX   rX   rY   r     s   &zKTOTemplateMixin.data_collatorr  r   r  rX   rX   rX   rY   r    s    

8r  )r  )r  r  r  	functoolsr   typingr   r   r   r   r   r   r	   r
   r   r   r  	packagingr   r   Z!transformers.dynamic_module_utilsr   Ztransformers.integrationsr   Z
modelscoper   baser   r   utilsr   r   r   r   r   r   r   r   r   r   r   r  r   r   ZHistoryr   PromptZ	StopWordsr   r    r   r   r!   r   r]   r   r   r   r   r   r   r"   r   r   ra   r_   rM   rf   r   r   r   rb   r`   r   r  r  rc   r  r  re   rd   r3  rz   Zyi_vl_default_systemr6  rZ  rb  r@   r{   r#   r$   r^   r%   r&   Zcodegeex4_systemr'   r;   r|   r<   r=   ZLLAMA_DEFAULT_SYSTEMr(   rg   rt   r  r  r  r*   r)   ZOPENBUDDY_DEFAULT_SYSTEMr-   ZOPENBUDDY2_DEFAULT_SYSTEMr.   r  r/   r  rh  r  r  r  r0   r  r  ru   r  rw   rv   r  r  r  r  r  r  r1   rx   r2   ry   r   r3   r5   r4   r6   r7   r8   r   r>   r}   r  r  rl   r  ro   rp   r(  r)  r,   r,  r+   r-  r4  r5  r6  rh   ri   r7  rj   r:  rk   r;  rm   r<  rn   r?  rs   rA  rD   rL  rH   rM  r   rO  rq   rP  rr   r(  rR  r~   r9   r:   rA   rY  r   r   r?   rf  r   rB   ri  rj  rx  r   r  r   r   Zgemma_templaterC   rI   r   r  r  rJ   rK   ZC4AI_SYSTEMrL   r  r   r  rE   r   Z_wizardlm2_systemrF   rG   r  r  rX   rX   rX   rY   <module>   sd  ,4 o#'T%- &$

,

h?L
W	 
%
-
"+'*6+*	KS"	&K(