o
    rqiv                     @   sl  d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
mZmZmZmZ d dlZd dlZd dlZd dlmZmZmZ d dlmZmZmZmZ d dlmZ d dlmZ d d	l m!Z!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0m1Z1m2Z2 d dl3m4Z4m5Z5 d dl6m7Z7 d dl8m9Z9 d dl:m;Z;m<Z<m=Z= e7 Z>dZ?G dd dZ@e(jAe2jBdde(jAe2jCddG dd dee;ZDe@Eddd ZFe@Gde@Gddd  ZHe@Ed!e@Ed"d#d$ ZIe@Ed%e@Ed&d'd( ZJe@Ed)d*d+ ZKe@Ed,d-d. ZLe@Edd/d0 ZMe@Ed1d2d3 ZNe@Ode! e@Od!e" e@Od"e" dS )4    N)contextmanager)Lock)	AnyCallableDict	GeneratorIteratorListOptionalTupleUnion)
AutoConfigPreTrainedModelPreTrainedTokenizer)AutoModelForCausalLMAutoTokenizerPipelinesnapshot_download)model_file_download)Model)ChatGLM2TokenizerLlama2Tokenizer)
OutputKeys)Input)	PIPELINES)is_modelis_official_hub_path)Config)
FrameworksInvoke	ModelFileTasks)create_devicedevice_placement)
get_logger)ModelTypeHelper)PipelineStreamingOutputMixinStreamingOutputMixinadd_stream_generateswiftc                   @   s   e Zd Zdg diZeddedefddZeddedefd	d
ZeddefddZ	eddefddZ
eddefddZededefddZededefddZdS )LLMAdapterRegistryZqwenNNNr   N
model_typevalue_indexc                 C   s\   |st jsJ |d u rt j}|| jvrg d| j|< | j| | d u s%J || j| |< |S )Nr+   )r%   Zcurrent_model_typellm_format_map)clsr,   r-   member r1   l/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/modelscope/pipelines/nlp/llm_pipeline.py_add_to_map(   s   
zLLMAdapterRegistry._add_to_mapc                    s*   |d ur  |S  fdd}|S )Nc                    s     | S Nr3   )r0   r/   r,   r-   r1   r2   	_register8   s   z.LLMAdapterRegistry._wrapper.<locals>._registerr5   )r/   r,   r-   r0   r7   r1   r6   r2   _wrapper3   s   zLLMAdapterRegistry._wrapperc                 C      |  |d|S )Nr   r8   r/   r,   functionr1   r1   r2   register_format_messages=      z+LLMAdapterRegistry.register_format_messagesc                 C   r9   )N   r:   r;   r1   r1   r2   register_format_outputA   r>   z)LLMAdapterRegistry.register_format_outputc                 C   r9   )N   r:   )r/   r,   tokenizer_classr1   r1   r2   register_tokenizerE   r>   z%LLMAdapterRegistry.register_tokenizer
model_namereturnc                 C   s
   || j v S r4   r.   r/   rD   r1   r1   r2   containsI      
zLLMAdapterRegistry.containsc                 C   s
   | j | S r4   rF   rG   r1   r1   r2   getM   rI   zLLMAdapterRegistry.get)r   N)NN)__name__
__module____qualname__r.   classmethodstrintr3   r8   r=   r@   rC   boolrH   rJ   r1   r1   r1   r2   r*   $   s     
	r*   Zllm)module_namec                       sF  e Zd Zdd Zdeeef defddZd<dd	Z		
	
	
	
	
d=dee
ef de
dededee f
 fddZd>ddZedeeef fddZdeeef fddZdeeee f defddZdeeef deeef deeef defdd Zdeeef fd!d"Zd#d$ Zd%d& Zd?d'd(Zed)eeeeeef  f dedeeejf fd*d+Zed,efd-d.Z ed/eeeeeef  f de!e"eef  fd0d1Z#ed2ee$ d3ed4eeeeeef  f dedee$ f
d5d6Z%eded4eeeeeef  f fd7d8Z&ed2ee$ d9ee$ee$ f dee$ fd:d;Z'  Z(S )@LLMPipelinec                 C   s  ddl m} t|trtd|  | |rT| jd ur'td| j d | j	
d}|d us5J d| j	
dd	}tj||tj| j| j| jd
}|j||d}|S t|trt|rtd| d | jrtj|rq|nt|}z| || j}td| j d |W S  ty } ztd| d| j d|  d | _W Y d }~nd }~ww t|rtj|tj| j| j| jdS tj|r|nt|}tj|| j| jd}||_|S |S )Nr   )Swiftzinitiate model from z.Cannot use swift with llm_framework, ignoring .zadapter_cfg.model_id_or_pathzECannot get adapter_cfg.model_id_or_path from configuration.json file.zadapter_cfg.model_revisionZmaster)
invoked_by
device_maptorch_dtypetrust_remote_code)model_idzinitiate model from location zinitiate model with z Cannot using llm_framework with z, ignoring llm_framework=z : )rV   rW   rX   ignore_file_pattern)rW   rY   )r)   rT   
isinstancerO   loggerinfo_is_swift_modelllm_frameworkwarningcfgsafe_getr   from_pretrainedr   ZPIPELINErW   rX   rY   r   ospathexistsr   _wrap_infer_framework	Exceptionr   r[   r   	model_dir)selfmodelrT   Z
base_modelrevisionZswift_modelrj   er1   r1   r2   initiate_single_modelV   s   


z!LLMPipeline.initiate_single_modelrl   rE   c                 C   sn   t |tsdS tj|rtj|tj}nzt|tj}W n
 t	y(   Y dS w t
|| _| jdtkS )NFzadapter_cfg.tuner_backend)r\   rO   re   rf   rg   joinr    CONFIGURATIONr   ri   r   	from_filerb   rc   SWIFT_FRAMEWORK)rk   rl   Zcfg_filer1   r1   r2   r_      s   
zLLMPipeline._is_swift_modelvllmc                 C   s   ddl m} |||S )Nr   )InferFramework)Z$modelscope.pipelines.accelerate.baseru   rd   )rk   rj   	frameworkru   r1   r1   r2   rh      s   z!LLMPipeline._wrap_infer_frameworkNformat_messagesformat_output	tokenizerr`   rY   c                    s  | dd | _|| _|| _tj|d r8tj|d | jd}|j	
dd }	|	r8|	
dddkr8tj r8d| _| dd | _| d	d | _|tkrX| |d |
d
d d S | | t j|i | W d    n1 sqw   Y  t| jtrt| j| _d }
t|trt|sJ d| dt
|\}}}
|d u rtj
| jjdd}t|rt
|\}}}
|d ur|| _|d ur|| _|d u r|  |
| _!d S || _!d S )NrW   rl   rY   Zquantization_configZquant_methodZgptqcudarX   r[   deviceZgpuzCan not find function for `z`!-)split)"poprW   rY   r`   re   rf   rg   r   rd   __dict__rJ   torchr{   Zdevice_countrX   r[   rs   _init_swift_temp_configuration_filesuper__init__r\   rl   r   r(   rO   r*   rH   r%   rj   rw   rx   _get_tokenizerry   )rk   rw   rx   ry   r`   rY   argskwargsconfigZq_configrB   r,   	__class__r1   r2   r      sd   




zLLMPipeline.__init__c                    s  ddl m} ddl m}m} dttttttf  f dtdtttj	f f fdd}dttttttf  f dtttf f fd	d
}||d}|| j
d\}	}
t|	 _ j|
_|
 _|
j _| _d _tj _| _t| _d _t  _d _d _d S )Nr   )prepare_model_template)InferArgumentsget_model_info_metamessagesry   rE   c                    s    j | }|dd  d|v r"t|d d  }||d< |jd }d|v r5|d d  }||d< |jd }t|d  |d< d|v rMt|d d  |d< |S )Nlabels	input_idsr?   inputs_embedsZattention_maskZtoken_type_ids)templateencoder   r   tensorshapeZones)r   ry   r   inputsr   Z	token_lenr   rk   r1   r2   rw      s$   

z0LLMPipeline._init_swift.<locals>.format_messagesc                    s   | d } t | dksJ dd }| d d dkr$| d d }| dd  } t | d dks0J d	d
d | D }|d }tt|d d d |dd d } jtkrXt|||dS t|||dS )Nr   r   zmessages cannot be empty!rolesystemcontentr?   rA   zUnsupported messages format!c                 S   s   g | ]}|d  qS )r   r1   ).0messager1   r1   r2   
<listcomp>   s    z@LLMPipeline._init_swift.<locals>.get_example.<locals>.<listcomp>)r   queryhistory)r   promptr   )lenlistzipr`   rs   dict)r   r   contentsr   r   r   r1   r2   get_example   s   "
z,LLMPipeline._init_swift.<locals>.get_example)rl   )rW   FT)Z	swift.llmr   r   r   r   rO   r	   r   r   TensorrW   r(   rl   r   ry   rw   has_multiple_modelsr   rv   device_namer"   r|   _model_preparer   Z_model_prepare_lock_auto_collate_compile)rk   rZ   r|   r   r   r   rw   r   r   rl   r   r1   r   r2   r      s:   





zLLMPipeline._init_swiftr   c                 c   s    |  |d  |d< }t|tr|n|j}tj|tj}tj	|r)d V  d S t
|d}tddd| W d    n1 sBw   Y  d V  t| d S )Nrl   wZpytorchchat)rv   task)ro   r\   rO   rj   re   rf   rp   r    rq   rg   openjsondumpremove)rk   r   rl   rj   Zconfiguration_pathfr1   r1   r2   r     s   
z$LLMPipeline._temp_configuration_filec           
      O   s2  | di }| di }| di }t|tod|v  |d< |d< | j|fi |}| jd tfv rat| jdrC| jjdi ||}n4t| jdr]t| jjdr]| jjjdi ||}nt	dt
|d	   g}| j|fi |d
 }| jd tfv r| d
 t|d	 d
 d  }| j|fi |}	|	S )Npreprocess_paramsforward_paramspostprocess_paramsr   is_messagesgeneraterl   z"model does not support `generate`!r   r   r1   )rJ   r\   r   
preprocessr`   rs   hasattrrl   r   
ValueErrorr   flattennumpytolistr   postprocess)
rk   r   r   r   r   r   r   tokensoutputsresponser1   r1   r2   _process_single"  s(    zLLMPipeline._process_singler   c           
         s   t jts
J djsjrjd rjs  jdi |\ }}t |to/d|v   d< |d< t |t	rX fdd|D }g }|D ]}|
||| qI|S | }	|	||}|S )Nz,pipeline.model must be StreamingOutputMixin!r   r   r   c                    s   g | ]} | qS r1   )_preprocess_with_check)r   ir   rk   r1   r2   r   L  s    
z/LLMPipeline.stream_generate.<locals>.<listcomp>r1   )r\   rl   r'   r   modelsr   Zprepare_model_sanitize_parametersr   r   append_stream_singler   )
rk   r   r   r   r   r   Zmodel_input_listoutputZelemodel_inputr1   r   r2   stream_generate>  s<   
zLLMPipeline.stream_generater   r   r   c              	   c   s    t | j| jh | jtjkr7t  | jr| |}| jj	di ||}W d    n1 s1w   Y  n| jj	di ||}|D ]#}|
 d t|d d d  }| j|fi |}| | |V  qDW d    d S 1 ssw   Y  d S )Nr   r   r1   )r#   rv   r   r   r   Zno_gradr   Z_collate_fnrl   r   r   r   r   Z_check_output)rk   r   r   r   streamoutr1   r1   r2   r   \  s.   


 
"zLLMPipeline._stream_singlec                    s   | d}|r| j|| jfi |}n| j|fddi|}| d|d< t| jdr0| jj n t| jdrCt| jjdrC| jjj nt| jdrLd	 ntd
 fdd| D S )Nr   return_tensorsptr   r   r|   rl   r`   cpuz'model does not have `device` attribute!c                    s*   i | ]\}}|t |r| n|qS r1   )r   Z	is_tensorto)r   kvr|   r1   r2   
<dictcomp>  s    z*LLMPipeline.preprocess.<locals>.<dictcomp>)r   rw   ry   r   rl   r|   r   items)rk   r   r   r   r   r1   r   r2   r   q  s"   


zLLMPipeline.preprocessc                 K   s   | d}t|ts-tjtjf}t||r t|jdkr |d }| j	j
|fddi|}n|}|r<| j|fi |}|S tj|i}|S )Nr   r?   r   Zskip_special_tokensT)r   r\   rO   r   r   npZndarrayr   r   ry   decoderx   r   ZTEXT)rk   r   r   r   Z
shape_typer   r1   r1   r2   r     s$   


zLLMPipeline.postprocessc                 K   s
   i |i fS )a  
        this method should sanitize the keyword args to preprocessor params,
        forward params and postprocess params on '__call__' or '_process_single' method
        considered to be a normal classmethod with default implementation / output

        Default Returns:
            Dict[str, str]:  preprocess_params = {}
            Dict[str, str]:  forward_params = {}
            Dict[str, str]:  postprocess_params = pipeline_parameters
        r1   )rk   Zgenerate_parameterr1   r1   r2   r     s   
z LLMPipeline._sanitize_parametersc                 C   s8   t | jtr
| j}n| jj}|d u rt}|j|| jdS )Nrz   )r\   rl   rO   rj   r   rd   rY   )rk   rB   rj   r1   r1   r2   r     s   zLLMPipeline._get_tokenizerr   c                 K   s>   g }t | D ]\}}t ||||}qdtj|gtjdiS )Nr   )Zdtype)rS   _message_iter_concat_with_special_tokensr   r   Zint64)r   ry   r   r   r   r   r1   r1   r2   rw     s   zLLMPipeline.format_messagesr   c                 K   s   |   } dd| di}|S )Nr   	assistantr   r   )strip)r   r   r   r1   r1   r2   rx     s   zLLMPipeline.format_outputdatac                 c   s&    | d D ]}|d |d fV  qd S )Nr   r   r   r1   )r   pairr1   r1   r2   r     s   zLLMPipeline._message_iteridsr   r   c              	   C   sF   |j }|j}|d}|| }t||}t| ||||||S )N
)Zim_start_idZ	im_end_idr   r   rS   _encode_concat)r   r   r   ry   Zim_startZim_endZnl_tokenr1   r1   r2   r     s   
z'LLMPipeline._concat_with_special_tokensc                 C   s   t |tr| | S g }|D ]K}| \\}}|dkrSt| dd}| j}|d }|d }	tt|dd}
t	|
|ks@J d|	g|t	|
  }t
|||
||}q|| | q|S )	Nimageimg_token_span   r?   rA   zutf-8)encodingzImage url is too long.)r\   rO   r   rstripr   getattrimg_start_idr   bytesr   rS   r   extend)ry   r   encodedr   modalvaluer   r   Z
img_end_idZ
img_pad_idZlist_int_urlZpad_idsr1   r1   r2   r     s2   
zLLMPipeline._encoder   c                 G   s.   |D ]}t |tr| | q| | q| S r4   )r\   r   r   r   )r   r   itemr1   r1   r2   r     s
   
zLLMPipeline._concat)rt   )NNNNN)rE   Nr4   ))rK   rL   rM   ro   r   rO   r   rQ   r_   rh   r   r   r
   r   r   r   r   r   r   r   r	   r   r   r   r   r   r   r   staticmethodr   r   rw   rx   r   r   r   rP   r   r   r   __classcell__r1   r1   r   r2   rS   R   s    =


5:










4rS   Zchatglm2c                 K   s&   dd }|| fi |}||dddS )Nc              	   [   s   d}| d } | d d dksJ dt dt| d dD ]}|d	|d d | | d
 | |d  d
 7 }q|dt| d d | d d
 7 }|S )N r   r   r   userz&chatglm2 does not have system messagesr?   rA   u    [Round {}]

问：{}

答：{}

r   u   [Round {}]

问：{}

答：r   )ranger   format)r   r   r   r   r1   r1   r2   build_chatglm2_prompt  s"   "z7chatglm2_format_messages.<locals>.build_chatglm2_promptFr   return_token_type_idsr   r1   )r   ry   r   r   r   r1   r1   r2   chatglm2_format_messages  s   r   Zchatglmc                 K   s*   |   } | dd} d| d}d|i}|S )Nu   [[训练时间]]u   2023年r   r   r   )r   replace)r   r   r   r   r1   r1   r2   chatglm2_format_output  s   
r  llamaZllama2c                 K   s6   ddl m} dd }|| |fi |\}}|d|iS )Nr   BatchEncodingc                 [   s  | dd}d}| d } | d d dkrd|dg|  } | d d	 }d
| d}||ddj}| d d	 }|  d}	||	ddj}
|jd |
jd  }||kr[td| d| d}g }tt| d ddD ]?}| | d	 | |d  d	 }}|  d|  d}||ddj}||jd  |kr n|| }|g| }||jd 7 }qi|||	g}|g| |
g }d|tj	|ddfS )N
max_length   zyou are a helpful assistant!r   r   r   r   r   r   z<s>[INST] <<SYS>>
z
<</SYS>>

r   r   r   z [/INST]zprepend prompt length z is bigger than max_length r   rA   r?   z	 [/INST] z </s><s>[INST] )dim)
rJ   r   r   r   RuntimeErrorr   r   rp   r   cat)r   ry   r   r  default_system_messager   system_promptZ
system_idstextZtext_promptZtext_idsZprompt_lengthZhistory_promptZhistory_ids_listr   r   r   Zround_promptZ	round_idsprompt_listZprompt_ids_listr1   r1   r2   build_llama2_prompt  sL   

z3llama2_format_messages.<locals>.build_llama2_promptr   )transformersr  )r   ry   r   r  r  r   r   r1   r1   r2   llama2_format_messages  s   ,r  ZbaichuanZ	baichuan2c                 K   sB  ddl m} ddd}| d } d}d}|d	d pd
}d}|| }	|| dd\}
}||
}|	t| }g }|d d d D ]?}g }|D ]}|d dkrQ|| n|| |||d  qCt|dksqt|t| |kr||| }t||k r|q= || }| d d dkr|| ||	 d  }t|g}|d|iS )Nr   r  r   c                 S   s   dg }}g }t | D ]+\}}|d dkr"|dksJ d|d }q|d |kr1|r1|| g }|| q|r>|| ||fS )Nr   r   r   r   'first message should be system message.r   )	enumerater   )r   
split_roler   roundsroundr   r   r1   r1   r2   _parse_messagesR  s   


z1baichuan_format_messages.<locals>._parse_messagesr         max_new_tokensr  i   )r  r   r   r   r   r   )r   )	r  r  rJ   r   r   r   r   r   Z
LongTensor)r   ry   r   r  r  Zassistant_token_idZuser_token_idr  Zmodel_max_lengthZmax_input_tokensr   r  Zsystem_tokensZmax_history_tokensZhistory_tokensr  Zround_tokensr   Zinput_tokensr1   r1   r2   baichuan_format_messagesM  sD   



r  Zwizardlmc                 K   s(   dd }|| |fi |}||dddS )Nc           
      [   s   d}	 | d } | d d dkrd|dg|  } | d d }|g}t | dd  D ]*\}}|d d	kr?|d }|d
|  q(|d dkrR|d }|d| d q(d|}	|	S )NzGA chat between a curious user and an artificial intelligence assistant.r   r   r   r   r   r   r?   r   zUSER: r   zASSISTANT: z</s> )r  r   rp   )
r   ry   r   r  r  r  r   r   Zuser_promptpromptsr1   r1   r2   build_wizardlm_prompt  s*   
z7wizardlm_format_messages.<locals>.build_wizardlm_promptFr   r   r1   )r   ry   r   r  r  r1   r1   r2   wizardlm_format_messages  s   r   Z
wizardcodec           	      K   s   | d } t | dksJ dd\}}t| D ](\}}|d dkr,|dks(J d|d	 }|d d
kr>|dks:J d|d	 }q|d | d }||ddddd}|S )Nr   rA   z&wizard code only support two messages.)r   r   r   r   r   r  r   r   r?   z&second message should be user message.z

### Instruction:
z

### Response:Fr   )r   paddingZadd_special_tokensr   )r   r  )	r   ry   r   r   r   r   r   r   r   r1   r1   r2   wizardcode_format_messages  s(   r"  c                 K   sT   | d } | d d | d d }}|j ||d}|j|d|dg}||d< |S )Nr   r   r   )r   z<|user|>z<|observation|>eos_token_id)Zbuild_chat_inputr#  Zget_command)r   ry   r   r   r   r   r#  r1   r1   r2   chatglm3_format_messages  s   r$  Zqwen2c                 K   s&   | d } |j | ddd}||gddS )Nr   FT)tokenizeZadd_generation_promptr   r  )Zapply_chat_template)r   ry   r   r  r1   r1   r2   qwen2_format_messages  s
   r&  )Pre   
contextlibr   	threadingr   typingr   r   r   r   r   r	   r
   r   r   r   r   r   r   r  r   r   r   Z
modelscoper   r   r   r   Zmodelscope.hub.file_downloadr   Zmodelscope.models.baser   Zmodelscope.models.nlpr   r   Zmodelscope.outputsr   Zmodelscope.pipelines.baser   Zmodelscope.pipelines.builderr   Zmodelscope.pipelines.utilr   r   Zmodelscope.utils.configr   Zmodelscope.utils.constantr   r   r    r!   Zmodelscope.utils.devicer"   r#   Zmodelscope.utils.loggerr$   Z"modelscope.utils.model_type_helperr%   Z!modelscope.utils.streaming_outputr&   r'   r(   r]   rs   r*   Zregister_moduler   Ztext_generationrS   r=   r   r@   r  r  r  r   r"  r$  r&  rC   r1   r1   r1   r2   <module>   sj   ,.   &

35



