o rq i"@sdZddlmZmZmZmZddlZddlZddlZddl Z ddl m Z ddl m Z mZmZmZddlZddlZddlmZmZddlmZeZdd iZGd d d eZdS) zTokenization classes for QWen.)absolute_importdivisionprint_functionunicode_literalsN)open)ListOptionalTupleUnion) AddedTokenPreTrainedTokenizer) get_logger vocab_file qwen.tiktokencseZdZdZ eZ         d+fdd Zd d Zd d Zd dZ de de e fddZ de de e fddZde e de fddZeddZdede fddZde defdd Zede e fd!d"Zede efd#d$Zd%d&Z d,d'eee efd(ede fd)d*ZZS)- QWenTokenizerzQWen tokenizer.replaceN <|endoftext|>FTc  st|tr t|dddn|}t|trt|dddn|}t|tr(t|dddn|}t|tr6t|dddn|}tj||||||| d| |_|durN|ntd|_||_d} d} d}d}| rv| ||d d d d d ft ddt dD}n| ||f}d}dtddfdd}||}ddt |t |dD}||_ tj| |||d}t |t ||jksJt |t |d|jd||_|j|_dd|jD|_||_|jj|_|||_|||_dS)NF)lstriprstrip)errors unk_token bos_token eos_token pad_tokenadd_prefix_space add_bos_tokengmBZQwenrz <|im_start|>z <|im_end|>zzzzzcSsg|]}d|dqS)z.0irrn/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/modelscope/models/nlp/qwen/tokenization.py Wsz*QWenTokenizer.__init__..zn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+tiktoken_bpe_filereturnzdict[bytes, int]cSs*t|d}dddd|DDS)NrbcSs i|] \}}t|t|qSr)base64 b64decodeint)rtokenZrankrrr! aszEQWenTokenizer.__init__..load_tiktoken_bpe..css|] }|r|VqdSN)split)rlinerrr! cszDQWenTokenizer.__init__..load_tiktoken_bpe..)rread splitlines)r$contentsrrr!load_tiktoken_bpe_s z1QWenTokenizer.__init__..load_tiktoken_bpecSi|]\}}||qSrr)rindexr*rrr!r+hsz*QWenTokenizer.__init__..)start)pat_strmergeable_ranksspecial_tokensz != z in encodingcSr4rr)rkvrrr!r+z) isinstancestrr super__init__rr)max_lenrtuplerange enumeratelenr9tiktokenZEncodingn_vocabr8encoderitemsdecoder tokenizerZ eot_tokenZeod_idZ im_start_idZ im_end_id)selfrrrArrrrrrZadd_more_sp_tokenskwargsnameZ ENDOFTEXTZIMSTARTZIMENDr9ZPAT_STRr3r8enc __class__rr!r@!s     zQWenTokenizer.__init__cC|jjSr,rKrGrLrrr!__len__szQWenTokenizer.__len__cCs|jSr,)r8rTrrr! get_vocabszQWenTokenizer.get_vocabcCsg}t|tr||jvr|j|S|j|S|D]}||jvr)||j|q||j|qt||jkrFt d t||j|S)NzToken indices sequence length is longer than the specified maximum sequence length for this model ({} > {}). Running this sequence through the model will result in indexing errors) r=r>r9rHgetappendrErAloggerwarningformat)rLtokensidsr*rrr!convert_tokens_to_idss     z#QWenTokenizer.convert_tokens_to_idssave_directoryr%cKstj|d}t|ddd)}|jD]\}}t|ddt |d}| |qWd|fS1s:wY|fS)z Save only the vocabulary of the tokenizer (vocabulary + added tokens). Returns: `Tuple(str)`: Paths to the files saved. rwutf8)encoding  N) ospathjoinrr8rIr' b64encodedecoder>write)rLr_rM file_pathr`r:r;r.rrr!save_vocabularys   zQWenTokenizer.save_vocabularytextcKs6g}td|}|j|D] }||j|q|S)a Converts a string in a sequence of tokens, replacing unknown tokens with the `unk_token`. Args: text (`str`): The sequence to be encoded. kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific encode method. See details in [`~PreTrainedTokenizerBase.__call__`] Returns: `List[str]`: The list of tokens. NFC) unicodedata normalizerKZencode_ordinaryrXrJ)rLrmrMr\trrr!tokenizes  zQWenTokenizer.tokenizer\cs0d|}tfdd|Djdjd}|S)z Converts a sequence of tokens in a single string. The most simple way to do it is `" ".join(tokens)` but we often want to remove sub-word tokenization artifacts at the same time. csg|]}j|qSr)Z byte_decoder)rcrTrr!r"r<z:QWenTokenizer.convert_tokens_to_string..zutf-8)r)rg bytearrayrir)rLr\rmrrTr!convert_tokens_to_strings z&QWenTokenizer.convert_tokens_to_stringcCrRr,rSrTrrr! vocab_sizeszQWenTokenizer.vocab_sizer5cCs ||jjkr |jS|j|gSr,)rKrGrri)rLr5rrr!_convert_id_to_tokens z"QWenTokenizer._convert_id_to_tokenr*cCs&|j|d|jj|jdddS)z*Converts a token to an id using the vocab.zUTF-8all)Zallowed_specialr)rHrWencoderKr)rLr*rrr!_convert_token_to_idsz"QWenTokenizer._convert_token_to_idcCdd|jD}|S)z `List[str]`: All the special tokens (`''`, `''`, etc.) mapped to class attributes. Convert tokens of `tokenizers.AddedToken` type to string. cSsg|]}t|qSr)r>)rsrrr!r"sz4QWenTokenizer.all_special_tokens..)r9keys)rLZall_toksrrr!all_special_tokenssz QWenTokenizer.all_special_tokenscCr|)zy `List[int]`: List the ids of the special tokens(`''`, `''`, etc.) mapped to class attributes. cSsg|]}|qSrr)rr;rrr!r"sz1QWenTokenizer.all_special_ids..)r9values)rLZall_idsrrr!all_special_idsszQWenTokenizer.all_special_idscKst)a Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces). Do NOT take care of added tokens. )NotImplementedError)rLrmrMrrr! _tokenizeszQWenTokenizer._tokenize token_idsskip_special_tokensc s2t|tr|g}|rfdd|D}j|S)Ncsg|] }|jvr|qSr)rrrTrr!r"sz)QWenTokenizer._decode..)r=r)rKri)rLrrrMrrTr!_decodes  zQWenTokenizer._decode) rNrrrNFFT)F)__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESZvocab_files_namesr@rUrVr^r>r rlrrrrvpropertyrwr)rxr{rrrr boolr __classcell__rrrPr!rsJ_   r)r __future__rrrrr'loggingreroiortypingrrr r jsonrFZ transformersr r Zmodelscope.utils.loggerr rYrrrrrr!s