o
    )i#
                     @   s   d dl mZmZmZmZ zd dlZd dlmZ W n ey#   edw ddl	m
Z
 ddlmZmZ ded	eeeeef  fd
dZded	efddZG dd dZddgZdS )    )ListSetTupleUnionN)ExLlamaV2TokenizerzJexllamav2 is not installed. Please install it with "pip install exllamav2"   )CharacterLevelParser)TokenEnforcerTokenEnforcerTokenizerData	tokenizerreturnc                 C   s   | j  }t| j }|| j| j| j| j	h | 
 }g }t|D ]}||v r*q#|| }t|dko9|d dk}||||f q#|S )Nr    )r   
vocab_sizesetZextended_id_to_piecekeysupdateZbos_token_ideos_token_idZpad_token_idZunk_token_idZget_id_to_piece_listrangelenappend)r   r   Zall_special_idsZid_to_pieceregular_tokensZ	token_idxdecodedZis_word_start_token r   s/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/lmformatenforcer/integrations/exllamav2.py_build_regular_tokens_list   s   
r   c                    s0   t  }dtt dtf fdd}t|| jS )Ntokensr   c                    s   t j| t jd} |S )N)Zdtype)torchtensorlongdecode)r   r   r   r   r   _decode   s   
z4build_token_enforcer_tokenizer_data.<locals>._decode)r   r   intstrr
   r   )r   r   r!   r   r    r   #build_token_enforcer_tokenizer_data   s   r$   c                   @   s   e Zd ZU dZee ed< dedee	e
f fddZdedd	fd
dZdejdd	fddZdd Zdeee ee f fddZd	S )ExLlamaV2TokenEnforcerFilterzkExLlamaV2Sampler.Settings.filters filter that uses the token enforcer to only allow format-complying tokenstoken_sequencecharacter_level_parsertokenizer_datac                 C   s(   t |tr	t|}t||| _g | _d S N)
isinstancer   r$   r	   token_enforcerr&   )selfr'   r(   r   r   r   __init__*   s   

z%ExLlamaV2TokenEnforcerFilter.__init__
prefix_strr   Nc                 C   s
   g | _ d S r)   )r&   )r,   r.   r   r   r   begin2   s   
z"ExLlamaV2TokenEnforcerFilter.begintokenc                 C   s   | j t|d d  d S )Nr   )r&   r   r"   )r,   r0   r   r   r   feed5   s   z!ExLlamaV2TokenEnforcerFilter.feedc                 C   s   | S r)   r   )r,   r   r   r   clone8   s   z"ExLlamaV2TokenEnforcerFilter.clonec                 C   s   | j | j}t|t fS r)   )r+   Zget_allowed_tokensr&   r   )r,   Zallowed_tokensr   r   r   next;   s   z!ExLlamaV2TokenEnforcerFilter.next)__name__
__module____qualname____doc__r   r"   __annotations__r   r   r   r
   r-   r#   r/   r   ZTensorr1   r2   r   r   r3   r   r   r   r   r%   &   s   
 

"r%   )typingr   r   r   r   r   Z	exllamav2r   ImportErrorZcharacterlevelparserr   Ztokenenforcerr	   r
   r"   r#   boolr   r$   r%   __all__r   r   r   r   <module>   s     
