o |i}N@sddlZddlmZmZddlZddlmZddlm Z m Z ddl m Z m Z mZddlmZddlmZmZmZmZmZmZmZddZeed d fd d Zeed d fd dZGdddZGdddeZe dedS)N)ListTuple)ImageImageOps) AutoProcessor BatchFeatureLlamaTokenizerFast)ProcessorMixin) IMAGE_SIZE BASE_SIZE CROP_MODE MIN_CROPS MAX_CROPSPROMPT TOKENIZERc Cs|td}d}||}|D]/}|d|d} t|| } | |kr%| }|}q | |kr;|d|||d|dkr;|}q |S)Ninfrrr?)floatabs) aspect_ratio target_ratioswidthheight image_sizeZbest_ratio_diff best_ratioarearatiotarget_aspect_ratioZ ratio_diffr U/home/app/DeepSeek-OCR/DeepSeek-OCR-master/DeepSeek-OCR-vllm/process/image_process.pyfind_closest_aspect_ratio s  r"Fc sN||}tfddtdD}t|ddd}t|||||}|S)Nc3X|]'}td|dD]}td|dD]}||kr||kr||fVqq qdSrNrange.0nijmax_nummin_numr r!  zcount_tiles..rcS|d|dSNrrr xr r r!$zcount_tiles..key)setr'sortedr") orig_width orig_heightr/r.r use_thumbnailrrrr r-r! count_tiless  r?cs4|j\}}||}tfddtdD}t|ddd}t|||||} || d} || d} | d| d} || | f} g}t| D].}|| |||| |||| |d||| |d|f}| |}||qKt|| ksJ|rt|dkr|||f}|||| fS)Nc3r$r%r&r(r-r r!r02r1z%dynamic_preprocess..rcSr2r3r r4r r r!r66r7z$dynamic_preprocess..r8r) sizer:r'r;r"resizecropappendlen)imager/r.rr>r<r=rrrZ target_widthZ target_heightblocksZ resized_imgZprocessed_imagesr+boxZ split_imgZ thumbnail_imgr r-r!dynamic_preprocess-s6         rHc@sNeZdZ   d deeeefdeeeefdefddZdejfd d Zd S) ImageTransformrrrTmeanstd normalizecCsB||_||_||_tg}|r|t||t||_dSN) rKrLrMTToTensorrC NormalizeCompose transform)selfrKrLrMZtransform_pipelinesr r r!__init__[s zImageTransform.__init__pil_imgcCs||}|SrN)rS)rTrVr5r r r!__call__js zImageTransform.__call__N)rJrJT) __name__ __module__ __qualname__rrboolrUrrWr r r r!rIYs   rIcsNeZdZdZdgZeddggddddddd d d dd f ded eeeefdededee e e fdee e e fde de de de de de deffdd Z e ddZe ddZe dd Zd5d!e d"e d#e fd$d%Zd&eed'e fd(d)Z d6d*e d+ed,e fd-d.Zdd/d*e d+ed,e fd0d1Z   d7d+eejd"e d#e d2e fd3d4ZZS)8DeepseekOCRProcessor)LlamaTokenizerr tokenizerirJTzu<｜▁pad▁｜>Fdeepseekicandidate_resolutions patch_sizedownsample_ratio image_mean image_stdrM image_token pad_tokenadd_special_token sft_format mask_prompt ignore_idc st|_t|_d|_||_||_||_d|_t |||d|_ ||_ d|j _ |j j dur2|j d| i|j j||_||_| |_ | |_| |_| |_| |_tj|fi|dS)Nr_r`)rKrLrMleftrh)r rr base_sizercrerfrMrdrIimage_transformr^ padding_siderhadd_special_tokensvocabgetimage_token_idrgrirjrkrlsuperrU)rTr^rbrcrdrerfrMrgrhrirjrkrlkwargs __class__r r!rUss0  zDeepseekOCRProcessor.__init__cC|jjSrN)r^ bos_token_idrTr r r!bos_idzDeepseekOCRProcessor.bos_idcCryrN)r^ eos_token_idr{r r r!eos_idr}zDeepseekOCRProcessor.eos_idcCryrN)r^ pad_token_idr{r r r!pad_idr}zDeepseekOCRProcessor.pad_idtextboseoscCs4|jj|dd}|r|jg|}|r||jg}|S)NF)rq)r^encoder|r)rTrrrtr r r!rs   zDeepseekOCRProcessor.encoderreturncKs|jj|fi|SrN)r^decode)rTrrvr r r!rszDeepseekOCRProcessor.decodepromptimagesinference_modec KsD|dur|dus Jd|}|d\}}}} } } } |||| | | dS)a Args: prompt (str): the formatted prompt; conversations (List[Dict]): conversations with a list of messages; images (List[ImageType]): the list of images; inference_mode (bool): if True, then remove the last eos token; system_prompt (str): the system prompt; **kwargs: Returns: outputs (BaseProcessorOutput): the output of the processor, - input_ids (torch.LongTensor): [N + image tokens] - target_ids (torch.LongTensor): [N + image tokens] - pixel_values (torch.FloatTensor): [n_patches, 3, H, W] - image_id (int): the id of the image token - num_image_tokens (List[int]): the number of image tokens Nz0prompt and images must be used at the same time.r) input_ids pixel_values images_cropimages_seq_maskimages_spatial_cropnum_image_tokensr ) rTrrrrvrjrrrrrr_r r r! process_onesz DeepseekOCRProcessor.process_one)rcKs|j|||d}|S)aJ Args: prompt (str): the formatted prompt; images (List[ImageType]): the list of images; inference_mode (bool): if True, then remove the last eos token; **kwargs: Returns: outputs (BaseProcessorOutput): the output of the processor, - input_ids (torch.LongTensor): [N + image tokens] - images (torch.FloatTensor): [n_images, 3, H, W] - image_id (int): the id of the image token - num_image_tokens (List[int]): the number of image tokens )rrr)r)rTrrrrvpreparer r r!rW*s zDeepseekOCRProcessor.__call__croppingc!Cst}||jt|ksJ||j}ggggf\}}} } g} g} g} t||D]\}} |j|ddd}| |7} | dgt|7} | |j|jddkr]|jddkr]ddg}n|rht |t d\}}nddg} |j dkr}|s}| |j |j f}t j||j|jftdd|jjDd }||| |\}}| ||g|dks|dkr tt|D] }||||q t|j |j|j}t|j|j|j}|jg||jg|}||jg7}|dks|dkr||jg|||jg||7}| |7} | d gt|7} | t|q) |j|d ddd}| |7} | dgt|7} |rD|jg| } dg| } |rR| |jg} | dg} t| t| ksiJd t| d t| g}| D]}||jkr|||qm||jqmt| t| krt|ksnJdt| dt|dt| dt| }t|}tj| tjd} |j||dk||jkB<|j ||dk<d }|r|d |jksJ|dd }|dd }| dd } t|dkrt!dd|j|jf}tj!dtj"d} t!dd|j |j f#d} n+tj$|dd}tj| tj"d} |r7tj$|dd#d} nt!dd|j |j f#d} |#d}||| | | | | ggS)z Tokenize text with tags.F)rrrr#r)rcss|] }t|dVqdS)N)int)r)r5r r r!r0sz.)colorTz2tokenize_with_images func: tokenized_str's length z) is not equal to imags_seq_mask's length ztokenized_str's length z, input_ids' length z, imags_seq_mask's length z, are not equal)dtypeNr)dim)%rcountrgrDsplitziprrCr@rHr rrArpadrntuplerorKr'mathceilrcrdrtr|rrltorch LongTensortensorr[rzeroslong unsqueezestack)!rTrrrr conversationZ text_splitsZ images_listZimages_crop_listrr image_shapesrZ tokenized_strZtext_seprEZ tokenized_sep crop_ratioZimages_crop_rawZ global_viewnum_width_tilesnum_height_tilesr+ num_queriesZnum_queries_baseZtokenized_imageZmasked_tokenized_str token_indexrZ target_idsrrrr r r!tokenize_with_imagesJs           (      z)DeepseekOCRProcessor.tokenize_with_images)TF)T)TTT)rXrYrZtokenizer_class attributesrrrrrr[strrUpropertyr|rrrrrrrWrr __classcell__r r rwr!r\os       e     > $r\ZDeepseekVLV2Processor)rtypingrrrZtorchvision.transforms transformsrOPILrr transformersrrrZtransformers.processing_utilsr configr r r r rrrr"r?rHrIr\registerr r r r!s"  $,