o
    0 iv                     @   s  d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ eeZi dddddddddddddddd
dddddddddddd
dddddddddddd
dddddddddddddd d!ddddddddddd"d#d$ddddddddddd
d%dddddddddddddd&d'ddddddddddd
d(d)d*d+d,d-d.ddddddddddd
d/d0dd1dd2d3d4d5d6d7d8d9d:dd2d;d<dd=d>dddddddd?dd@	dAd0dBdCdDd1d2dEdFdddddddGdHdIdddd2ddJdKdLddM	dddddddddNdd
dddddddddddOddPdddddddddddOddPd5d6d7d8d9d:dd2d;d<dd=ddddddddddd
dQZdRdSdTdUdVd)d*d+d,dWdX
dYdd)d*d+d,dZd[Zd\d] ZG d^d_ d_ZG d`da daeZ G dbdc dceZ!G ddde deeZ"G dfdg dgeZ#G dhdi dieZ$G djdk dkeZ%i de de!dle!d$e!d%e!d.e"d/e#d'e#d>e#dAe#dFe#d4e$dIe#dme#dne%doe%dpe$e e dqZ&dre'dse(ee)f fdtduZ*dS )vz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers
processors)BPEUnigram   
AddedToken)GemmaConverterGPT2ConverterLlamaConverterQwen2ConverterT5Converter)logging)tqdmZgeneralZ
model_typeZ_model_name_or_path)architecturenamellamaZmax_position_embeddingsZnum_hidden_layersZintermediate_sizeZhidden_sizeZhead_dimZ
rope_thetaZnum_attention_headsZnum_key_value_headsZrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonr   ZmistralZqwen2Zqwen2moeZnum_expertsZnum_experts_per_tok)r   r   r   r   r   r   r   r   r    r   expert_countexpert_used_countZlfm2Zconv_L_cache)r   r   r   r   r   r   r   r   r    r   zshortconv.l_cacheZqwen3Z	qwen3_moe)r   r   r   r   r   r   attention.key_lengthr   r   r    r   r!   r"   Zfalcon	tokenizerbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idZphi3ZbloomZn_layerZn_headZlayer_norm_epsilon)r   r   r   r   attention.layer_norm_epsilonZt5Zn_positionsZ
num_layersZd_ffZd_modelZd_kvZ	num_headsZrelative_attention_num_bucketsdecoder_start_token_id)r   r   r   r   r#   r   r   r-   z attention.relative_buckets_countr.   r   ZstablelmZlayer_norm_eps)	r   r   r   r   r   r   r   r-   r   Zgpt2Zn_ctxZn_embdr   )r   r   r   r   r   r-   Z
starcoder2Znorm_epsilon)r   r   r   r   r   r   r-   ZmambaZconv_kernelZ
state_sizeZtime_step_rank)	r   r   r   r    r   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeZnorm_epsZsliding_window)r   r   r   r   r   r   r#   r   r   r    zattention.sliding_windowr   )nemotrongemma2Zgemma3umt5decitokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr)   r*   r+   r,   zggml.add_space_prefixchat_template)r:   r9   r)   r*   r+   r,   )r$   Ztokenizer_configc                 C   s   t |ts|g}t|dkr|d }d }n|d dkrtd|\}}|dv r/t| d } | S |dv r;t| d } | S |dkrGt| d } | S |dkrXtd	t|  	 } | S |dkrat
| |} | S )
N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   r;   r
            
      )            B)
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_valueZ	data_typeZarray_data_type rR   j/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/transformers/integrations/ggml.pyrP   =  s.   
	
rP   c                   @   s   e Zd Zdd ZdS )GGUFTokenizerSkeletonc                    s  |  D ]
\}}t| || qt| dst| drt| ds"td| j}| j  fddt|D td g }t	  D ]=\}}g }t
dt|D ]}	|d |	 ||	d  }
}|
|v rl||v rl||
||f qMt|fd	d
dd}|| q@t|dd
 dd}dd |D }|| _ndd | jD | _t| dsdd t
t| jD | _t| dsg | _t| dsd | _t| dr| jd u r| j| _d S d S d S )Nr7   r4   r5   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                    s   i | ]	\}}| | qS rR   rR   ).0it)r5   rR   rS   
<dictcomp>a      z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>z:Merges were not in checkpoint, building merges on the fly.r;   c                    s    | d   | d  fS )Nr   r;   rR   )x)vocabrR   rS   <lambda>k  s    z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>T)keyreversec                 S   s   | d S )Nr
   rR   )valrR   rR   rS   r\   m  s    c                 S   s   g | ]
}|d  |d fqS )r   r;   rR   )rU   r_   rR   rR   rS   
<listcomp>n  s    z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>c                 S   s   g | ]	}t |d qS ) )tuplesplit)rU   mergerR   rR   rS   r`   q  rY   c                 S   s   g | ]}d qS NrR   )rU   _rR   rR   rS   r`   s  s    added_tokensr'   unknown_token_id)itemssetattrhasattrrJ   r4   r5   	enumerateloggerwarningr   rangerI   appendsortedextendr7   rg   r'   rh   )selfZdict_kvr4   r7   rd   Zpiece_scorelocalindexZpiece_lZpiece_rrR   )r5   r[   rS   __init__V  sD   




zGGUFTokenizerSkeleton.__init__N)__name__
__module____qualname__rx   rR   rR   rR   rS   rT   U  s    rT   c                   @   s<   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdS )GGUFLlamaConverterc                 C   s0   t || _| j| _i | _t| jdddk| _d S )Nr3   r   )rT   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerrs   tokenizer_dictrR   rR   rS   rx     s   
zGGUFLlamaConverter.__init__c                 C      t t|j|jS re   rH   zipr4   r5   rs   r}   rR   rR   rS   r[        zGGUFLlamaConverter.vocabc                 C      |j S re   r7   r   rR   rR   rS   r7        zGGUFLlamaConverter.mergesc                 C   s  |  | j}| | j}dd t|D }|jd ur |j|j nd }t|dd d ur0|j|j nd }t|dd d ur@|j|j nd }tt	|||ddd}g }	t
| jds|d urc|	t|ddd	 |d urq|	t|ddd	 |d ur|	t|ddd	 n!tt| jjd
kd }
|
D ]}|	t| jj| ddd	 qt|	dkr||	 t| jjdkr|dd | jjD  || jd< || jd< || jd< | jrd | jd< d| jd< d| jd< d| j_|S )Nc                 S      i | ]	\}\}}||qS rR   rR   rU   rV   wordZ_scorerR   rR   rS   rX     rY   z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>r%   r&   T)	unk_tokenZfuse_unkbyte_fallbackr6   F
normalizedspecialr=   r   c                 S   s   g | ]	}t |d d dqS )Fr   r   )rU   Zadded_tokenrR   rR   rS   r`     rY   z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>r   	eos_token	bos_tokenr8   Zclean_up_tokenization_spaceslegacy)r[   r}   r7   rl   r'   r4   r   r%   r   r   rk   rp   r   npwherer   r6   rI   add_special_tokensrg   Z
add_tokensr   r   r~   r   )rs   r}   vocab_scoresr7   	bpe_vocabr   r   r   r$   special_tokensZspecial_tokens_idxidxrR   rR   rS   r$     sT     







zGGUFLlamaConverter.tokenizerc                 C   sX   t  t  t ddg}| jr|t jddddg7 }|r'|t jdddg7 }t |S )N   ▁ra   FTr8   Ztrim_offsets	use_regexr;   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencers   replacementr8   sequencerR   rR   rS   decoder  s   

zGGUFLlamaConverter.decoderc                 C   s   |  | j}| | j}|d ur||_d}d}t| jdr!| jj}| ||}|d ur.||_| |||_|  }|r>||_| j	rPt
jdddd|_tg |_|S )Nr   Tr8   Fr   )r$   r}   
normalizerrk   r~   r8   pre_tokenizerr   post_processorr   r   r   r   r   )rs   r$   r   r   r8   r   r   rR   rR   rS   	converted  s*   zGGUFLlamaConverter.convertedN)	ry   rz   r{   rx   r[   r7   r$   r   r   rR   rR   rR   rS   r|     s    :r|   c                       *   e Zd Zdd Zdef fddZ  ZS )GGUFQwen2Converterc                 C      t || _i | _d S re   rT   r~   r   r   rR   rR   rS   rx        

zGGUFQwen2Converter.__init__returnc              	      s^   dd t | jjD }| jj}t ||}|tddddtddddtddddg |S )	Nc                 S      i | ]\}}||qS rR   rR   rU   rV   r   rR   rR   rS   rX         z0GGUFQwen2Converter.converted.<locals>.<dictcomp><|endoftext|>FTr   z<|im_start|>z
<|im_end|>)rl   r~   r4   r7   superr   r   r   rs   r[   r7   r$   	__class__rR   rS   r     s   zGGUFQwen2Converter.convertedry   rz   r{   rx   r   r   __classcell__rR   rR   r   rS   r         r   c                   @   sB   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdefddZ	dS )GGUFPhi3Converterc                 C   s   t || _| j| _i | _d S re   rT   r}   r~   r   r   rR   rR   rS   rx     s   

zGGUFPhi3Converter.__init__c                 C   r   re   r   r   rR   rR   rS   r[     r   zGGUFPhi3Converter.vocabc                 C   r   re   r   r   rR   rR   rS   r7     r   zGGUFPhi3Converter.mergesc                 C   sn  |  | j}| | j}dd t|D }tt||}|tddddddtddddtd	dddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
tddddd
g |jd ur|j	|j nd | j
d< |jd ur|j	|j nd | j
d< |jd ur|j	|j nd | j
d< |jd ur|j	|j nd | j
d< |S )Nc                 S   r   rR   rR   r   rR   rR   rS   rX     rY   z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp></s>TF)rstriplstripr   r   r   r   z<|assistant|>)r   r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   Z	pad_token)r[   r}   r7   rl   r   r   r   r   r'   r4   r   r&   r%   r(   )rs   r}   r   r7   r   r$   rR   rR   rS   r$     s8   zGGUFPhi3Converter.tokenizerc                 C   s<   t  t  t |dg}|r|t jdddg7 }t |S )Nra   r;   r   )r   r   r   r   r   r   r   rR   rR   rS   r   @  s   

zGGUFPhi3Converter.decoderr   c                 C   s:   |  | j}d}d}t| jdr| jj}| |||_|S )Nr   Tr8   )r$   r}   rk   r~   r8   r   )rs   r$   r   r8   rR   rR   rS   r   K  s   zGGUFPhi3Converter.convertedN)
ry   rz   r{   rx   r[   r7   r$   r   r   r   rR   rR   rR   rS   r     s    'r   c                       r   )GGUFGPTConverterc                 C   r   re   r   r   rR   rR   rS   rx   Y  r   zGGUFGPTConverter.__init__r   c                    s0   dd t | jjD }| jj}t ||}|S )Nc                 S   r   rR   rR   r   rR   rR   rS   rX   ^  r   z.GGUFGPTConverter.converted.<locals>.<dictcomp>)rl   r~   r4   r7   r   r   r   r   rR   rS   r   ]  s   zGGUFGPTConverter.convertedr   rR   rR   r   rS   r   X  r   r   c                   @   :   e Zd Zdd Zdd Zdd Zdd Zd	efd
dZdS )GGUFT5Converterc                 C   s>   dg|d< t || _dd t| jjD | _| j| _i | _d S )N
dummy textr7   c                 S   r   rR   rR   )rU   ru   rt   rR   rR   rS   rX   j  r   z,GGUFT5Converter.__init__.<locals>.<dictcomp>)rT   r}   rl   r4   token2idr~   r   r   rR   rR   rS   rx   e  s
   


zGGUFT5Converter.__init__c                 C   r   re   r   r   rR   rR   rS   r[   n  r   zGGUFT5Converter.vocabc                 C   sT   t | jddr(g }t | jddr|tjddg7 }|tjdddg7 }t|S d S )Nr   Tr8   r   )prependra   )patternr   )r   r~   r   ZPrependr   r   )rs   r}   r   rR   rR   rS   r   q  s   
zGGUFT5Converter.normalizerc                 C   s$   t jddgg dd| jd fgdS )N$Ar   )r   r   z$Br   )singlepairr   )r   ZTemplateProcessingr   )rs   rR   rR   rS   r   z  s   zGGUFT5Converter.post_processorr   c                 C   s   |  | j}tt|| jjdd}| | j}|d ur||_d}d}t| jdr,| jj}| 	||}|d ur9||_	| 
|||_
|  }|rI||_|S )NFZunk_idr   r   Tr8   )r[   r}   r   r	   r'   r   rk   r~   r8   r   r   r   )rs   r   r$   r   r   r8   r   r   rR   rR   rS   r     s.   	zGGUFT5Converter.convertedN)	ry   rz   r{   rx   r[   r   r   r   r   rR   rR   rR   rS   r   d  s    			r   c                   @   r   )GGUFGemmaConverterc                 C   s&   dg|d< t || _| j| _i | _d S )Nr   r7   r   r   rR   rR   rS   rx     s   


zGGUFGemmaConverter.__init__c                 C   s   t t|j|j}g }|D ]1\}}|dkr|d|f qd|v r7t| dkr7dt| }|||f q|||f q|S )Nz<0x09>	ra   r   r   )rH   r   r4   r5   rp   rI   strip)rs   r}   Zoriginal_vocabZupdated_vocabtokenZscoreZunderscoresrR   rR   rS   r[     s   zGGUFGemmaConverter.vocabc                 C   s   t ddS )Nra   r   )r   r   r   rR   rR   rS   r     s   zGGUFGemmaConverter.normalizerc                 C   s<   t ddt  t  g}|r|t jdddg7 }t |S )Nr   ra   r;   r   )r   r   r   r   r   r   r   rR   rR   rS   r     s   

zGGUFGemmaConverter.decoderr   c                 C   s   |  | j}tt|| jj| jd}| | j}|d ur||_d}d}t| jdr-| jj	}| 
|||_
| ||}|d urA||_|S )Nr   r   Tr8   )r[   r}   r   r	   r'   Zhandle_byte_fallbackr   rk   r~   r8   r   r   )rs   r   r$   r   r   r8   r   rR   rR   rS   r     s(   zGGUFGemmaConverter.convertedN)	ry   rz   r{   rx   r[   r   r   r   r   rR   rR   rR   rS   r     s    r   Z	qwen2_moer/   r0   Zgemma3_textr1   )r2   Zdecilmr   r   c                 C   s"   | }t | |}| }||jfS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr   r   )r   r   Ztokenizer_class_name	converterZfast_tokenizerrR   rR   rS   convert_gguf_tokenizer  s   
r   )+__doc__r   numpyr   Z
tokenizersr   r   r   r   r   Ztokenizers.modelsr   r	    r   Zconvert_slow_tokenizerr   r   r   r   r   utilsr   Zutils.loggingr   Z
get_loggerry   rm   ZGGUF_CONFIG_MAPPINGZGGUF_TOKENIZER_MAPPINGrP   rT   r|   r   r   r   r   r   r   strrb   dictr   rR   rR   rR   rS   <module>   s  
+9FRams       '  0  <   +yK?A	
