o
    rqiu                     @   sz  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZ d dlZd dlZd dlZd dlZd dlmZ dd	lmZ G d
d dZG dd deZdejdejfddZd'ddZ dd Z!		d(dejfddZ"		d)dejfddZ#		d*dejfddZ$d+dd Z%		d,d!d"Z&	d+d#d$Z'G d%d& d&Z(dS )-    N)OrderedDict)Mapping)Path)FunctionType)AnyDictUnion)nn   )compare_arguments_nestedc                   @   s   e Zd ZdZ			ddededefddZdd	 Zd
d Ze	j
		ddejdefddZe	j
						ddeeef fddZdS )RegressToolzThis class is used to stop inference/training results from changing by some unaware affections by unittests.

    Firstly, run a baseline test to create a result file, then changes can be observed between
    the latest version and the baseline file.
    Nbaseline
store_func	load_funcc                 C   s(   || _ || _|| _tdt   dS )zPA func to store the baseline file and a func to load the baseline file.
        zCurrent working dir is: N)r   r   r   printr   cwd)selfr   r   r    r   j/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/modelscope/utils/regress_test_utils.py__init__!   s   zRegressTool.__init__c                 C   s`   | j d ur|  || d S tjtjt ddd}tj|dd t	|tj|| d S )Ndatatest
regressionT)exist_ok)
r   ospathabspathjoinr   r   makedirsshutilcopy)r   localremoter   r   r   r   store,   s   
zRegressTool.storec              	   C   s   | j d ur|  || d S tjtjt ddd}tj||}tj|s1td| dt	d| dt
t|d    tj|rPt| tj||d	d
 d S )Nr   r   r   zbase line file z
 not existzlocal file found:z, md5:rbF)target_is_directory)r   r   r   r   r   r   r   exists
ValueErrorr   hashlibmd5openread	hexdigestremovesymlink)r   r!   r"   r   r   r   r   r   load5   s   
"
zRegressTool.loadTmodule	file_namec                 +   s,   t d}|du s| jdu rdV  dS | j}i }d  d}t|tjs/t|ds,J |j}t| | t	|| dV  t|dddd t	|ddd |ryt
|d}	t||	 W d   n1 sdw   Y  | |  d t | dS t j|}
t jt |
}| ||
 t
|d	}	t|	}W d   n1 sw   Y  G d
d dtj}|rtd  fdd| D }| D ]\}}ddd|d< qƇ fdd| D }| D ]\}}ddd|d< qtdtj||d  tdtj||d  t|||fi |stddS )a  Monitor a pytorch module in a single forward.

        Args:
            module: A torch module
            file_name: The file_name to store or load file
            compare_fn: A custom fn used to compare the results manually.
            compare_model_output: Only compare the input module's output, skip all other tensors

        >>> def compare_fn(v1, v2, key, type):
        >>>     return None

        v1 is the baseline value
        v2 is the value of current version
        key is the key of submodules
        type is in one of 'input', 'output'

            kwargs:
            atol: The absolute gap between two np arrays.
            rtol: The relative gap between two np arrays.
        REGRESSION_BASELINEN./.binmodelTrestorewbr$   c                   @      e Zd Zdd Zdd ZdS )zCRegressTool.monitor_module_single_forward.<locals>.SafeNumpyEncoderc                 S   sJ   t |tjr
| S t |tjrt|S t |tjrt|S tj	
| |S N)
isinstancenpZndarraytolistZfloatingfloatintegerintjsonJSONEncoderdefaultr   objr   r   r   parse_default   s   zQRegressTool.monitor_module_single_forward.<locals>.SafeNumpyEncoder.parse_defaultc                 S   s4   z|  |W S  ty   td|j d Y d S w )NzType z! cannot be serialized and printed)rC   	Exceptionr   	__class__rD   r   r   r   rC      s   zKRegressTool.monitor_module_single_forward.<locals>.SafeNumpyEncoder.defaultN)__name__
__module____qualname__rF   rC   r   r   r   r   SafeNumpyEncoder}   s    rL   zDIgnore inner modules, only the output of the model will be verified.c                       i | ]\}}| kr||qS r   r   .0keyvaluer1   r   r   
<dictcomp>       z=RegressTool.monitor_module_single_forward.<locals>.<dictcomp>argskwargsinputc                    rM   r   r   rN   rR   r   r   rS      rT   z
baseline: )clsz
latest  : zResult not match!)r   getenvr   r;   r	   Modulehasattrr5   hack_forwardintercept_moduler*   pickledumpr#   r-   r   basenamer   tempfile
gettempdirr/   rA   rB   r   itemsdumpscompare_io_and_printr'   )r   r0   r1   
compare_fnZcompare_model_outputrW   r   io_jsonabsolute_pathfnamebaserL   rP   rQ   r   rR   r   monitor_module_single_forwardE   s`   



z)RegressTool.monitor_module_single_forwardconfigtrainerc	              
   +   s   t d}
|
du s| jdu rdV  dS | j}
i }i }d| d}|dkr)td t|ds6d|v s6J dt|tr?|d n|j}t|tj	sRt|dsOJ |j}t|d	s_d	|v s_J d
t|dsld|v slJ dt|tru|d	 n|j
}t|tr|d n|j}tt }tj }t }t|dr|jn	t|dr|jnd}|rt   fdd  | W d   n1 sw   Y  |dkrt||| t|| t||||d dV  t||ddd |dkrt|dddd t|ddd | }|dd |||jj|j|d|jj| dt|dr!|j nd||||dd}|
rZt |d}t!"|| W d   n	1 sEw   Y  | #|| d t $| dS t j%&|}t j%'t() |}
| *|
| t |
d}t!*|}W d   n	1 sw   Y  |dkrt+|d ||fi |	st,dt-|d |f|||d|	st,d |d	 |d |d |sdn|d d!}|d	 |d |d |sdn|d d!}t.|||fi |	st,d"dS )#ay  Monitor a pytorch module's backward data and cfg data within a step of the optimizer.

        This is usually useful when you try to change some dangerous code
        which has the risk of affecting the training loop.

        Args:
            trainer: A dict or an object contains the model/optimizer/lr_scheduler
            file_name: The file_name to store or load file
            level: The regression level.
            'strict' for matching every single tensor.
                     Please make sure the parameters of head are fixed
                     and the drop-out rate is zero.
            'config' for matching the initial config, like cfg file, optimizer param_groups,
                     lr_scheduler params and the random seed.
            'metric' for compare the best metrics in the evaluation loop.
            compare_fn: A custom fn used to compare the results manually.
            ignore_keys: The keys to ignore of the named_parameters.
            compare_random: If to compare random setttings, default True.
            reset_dropout: Reset all dropout modules to 0.0.
            lazy_stop_callback: A callback passed in, when the moniting is over, this callback will be called.
            kwargs:
            atol: The absolute gap between two np arrays.
            rtol: The relative gap between two np arrays.

        >>> def compare_fn(v1, v2, key, type):
        >>>     return None

        v1 is the baseline value
        v2 is the value of current version
        key is the key of modules/parameters
        type is in one of 'input', 'output', 'backward', 'optimizer', 'lr_scheduler', 'cfg', 'state'
        r2   Nr3   r4   strictz[Important] The level of regression is 'strict', please make sure your model's parameters are fixed and all drop-out rates have been set to zero.r5   zmodel must be in trainer	optimizerzoptimizer must be in trainerlr_schedulerzlr_scheduler must be in trainer_seedseedc                    sB   |   D ]\}}t|tjjrt| |tjd q | qd S )Ng        )named_childrenr;   torchr	   ZDropoutsetattr)_modulerk   	submodulereinit_dropoutr   r   r{     s
   
z8RegressTool.monitor_module_train.<locals>.reinit_dropout)lazy_stop_callbackTr6   state)typedefaults
state_dict)r~   r   cfg)torch_statenp_staterandom_seedrt   )forwardbackwardrq   rr   r   r}   r8   r$   r   zForward not match!r   )rg   ignore_keyslevelzBackward not match!)rq   rr   r   r}   zCfg or optimizers not match!)/r   rZ   r   r   r\   r;   dictr5   r	   r[   rq   rr   numpify_tensor_nestedrv   Zget_rng_stater<   randomZ	get_stategetstaters   rt   Zno_gradr]   r^   hack_backwardr   poprH   rI   r   r   to_dictr*   r_   r`   r#   r-   r   ra   r   rb   rc   r/   rf   RuntimeErrorcompare_backward_and_printcompare_cfg_and_optimizers)r   ro   r1   r   rg   r   compare_randomZreset_dropoutr|   rW   r   rh   bw_jsonri   r0   rq   rr   r   r   r   rt   Zoptimizer_dictsummaryrj   rk   baseline_jsonZcfg_opt1Zcfg_opt2r   rz   r   monitor_module_train   s   
+





z RegressTool.monitor_module_train)NNNNT)rn   NNTTN)rI   rJ   rK   __doc__boolr   r   r#   r/   
contextlibcontextmanagerr	   r[   strrm   r   r   r   r   r   r   r   r   r      s@    
	d
r   c                   @   s6   e Zd ZG dd deZej					dddZdS )	MsRegressToolc                   @   s   e Zd ZdS )zMsRegressTool.EarlyStopErrorN)rI   rJ   rK   r   r   r   r   EarlyStopErrorV  s    r   rn   NTc           
      +   sV    d u rfdd fdd}	j tj |	__ d V  d S )Nc                     s   G dd d}   |   d S )Nc                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0S )1zQMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHookZ   c                 S      d S r:   r   r   ro   r   r   r   
before_runk     z\MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_runc                 S   r   r:   r   r   r   r   r   	after_runn  r   z[MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_runc                 S   r   r:   r   r   r   r   r   before_epochq  r   z^MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_epochc                 S   r   r:   r   r   r   r   r   after_epocht  r   z]MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_epochc                 S   r   r:   r   r   r   r   r   before_iterw  r   z]MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_iterc                 S      |  | d S r:   r   r   r   r   r   before_train_epochz     zdMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_train_epochc                 S   r   r:   r   r   r   r   r   before_val_epoch}  r   zbMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_val_epochc                 S   r   r:   r   r   r   r   r   after_train_epoch  r   zcMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_train_epochc                 S   r   r:   r   r   r   r   r   after_val_epoch  r   zaMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_val_epochc                 S   r   r:   r   r   r   r   r   before_train_iter  r   zcMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_train_iterc                 S   r   r:   r   r   r   r   r   before_val_iter  r   zaMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.before_val_iterc                 S   r   r:   
after_iterr   r   r   r   after_train_iter  r   zbMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_train_iterc                 S   r   r:   r   r   r   r   r   after_val_iter  r   z`MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_val_iterc                 S      |dkr|j d | dkS dS Nr   r
   F)epochr   ro   nr   r   r   every_n_epochs     z`MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.every_n_epochsc                 S   r   r   )
inner_iter)r   runnerr   r   r   r   every_n_inner_iters  s   zeMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.every_n_inner_itersc                 S   r   r   )iterr   r   r   r   every_n_iters  r   z_MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.every_n_itersc                 S      |j d |jkS Nr
   )r   Ziters_per_epochr   r   r   r   end_of_epoch     z^MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.end_of_epochc                 S   r   r   )r   Z
max_epochsr   r   r   r   is_last_epoch  r   z_MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.is_last_epochc                 S   r   r   )r   Z	max_itersr   r   r   r   is_last_iter  r   z^MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.is_last_iterc                 S   s   g S r:   r   r   r   r   r   get_triggered_stages  r   zfMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.get_triggered_stagesc                 S   s   i S r:   r   r   r   r   r   r     r   z\MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.state_dictc                 S   r   r:   r   )r   r   r   r   r   load_state_dict  r   zaMsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.load_state_dictc                 S   s
   t d)NzTest finished.)r   r   r   r   r   r   r     s   
z\MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callback.<locals>.EarlyStopHook.after_iterN)rI   rJ   rK   ZPRIORITYr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   EarlyStopHookh  s2    r   )register_hook)r   )ro   r   r   r|   f  s   Iz:MsRegressTool.monitor_ms_train.<locals>.lazy_stop_callbackc              	      s|   j | f d$ z| j|i |W W  d    S  tjy+   Y nw W d    d S 1 s7w   Y  d S )N)rg   r   r   r|   )r   train_loop_originr   r   )ro   Z
args_trainZkwargs_train)rg   r   r1   r   rW   r|   r   r   r   r   _train_loop  s,   	"z3MsRegressTool.monitor_ms_train.<locals>._train_loop)Z
train_loopr~   r   )
r   ro   r1   r   rg   r   r   r|   rW   r   r   )	rg   r   r1   r   rW   r|   r   r   ro   r   monitor_ms_trainY  s   M
zMsRegressTool.monitor_ms_train)rn   NNTN)rI   rJ   rK   rG   r   r   r   r   r   r   r   r   r   T  s    r   module1module2c                 C   s<   t |  | D ]\}}|j|j dkr dS q	dS )Nr   FT)zip
parametersr   nesum)r   r   p1p2r   r   r   compare_module  s
   r   '  c                    s  zddl m} W n ty   t}Y nw 	 t| t|fr*t fdd|  D S t| tr;t fdd| D S t| t	rLt	 fdd| D S t| t
jr|   } d urpt| k |}t|  k   |}dkrz|jtd	S d
kr|jtd	S |S | S )Nr   ModelOutputBasec                    s   i | ]\}}|t | qS r   r   rO   kt
clip_value	reductionr   r   rS     s    z)numpify_tensor_nested.<locals>.<dictcomp>c                 3       | ]	}t | V  qd S r:   r   rO   r   r   r   r   	<genexpr>      
z(numpify_tensor_nested.<locals>.<genexpr>c                 3   r   r:   r   r   r   r   r   r     r   r   )Zdtypemean)modelscope.outputsr   ImportErrorr   r;   r   r   rd   listtuplerv   Tensorcpunumpyr<   wherer   r>   r   )tensorsr   r   r   r   r   r   r   r     s:   

r   c                 C   s   zddl m} W n ty   t}Y nw 	 t| t|fr'tdd |  D S t| tr5tdd | D S t| t	rCt	dd | D S t| t
jrM|  S | S )Nr   r   c                 S   s   i | ]	\}}|t |qS r   detach_tensor_nestedr   r   r   r   rS     s    z(detach_tensor_nested.<locals>.<dictcomp>c                 s       | ]}t |V  qd S r:   r   r   r   r   r   r         z'detach_tensor_nested.<locals>.<genexpr>c                 s   r   r:   r   r   r   r   r   r     r   )r   r   r   r   r;   r   r   rd   r   r   rv   r   detach)r   r   r   r   r   r     s&   

r   Fr0   c                    s`    fdd}|st | ds| jt| j|| | _| _|r,t | dr.| j| _| `d S d S d S )Nc                    s   | j |i |}rtt|}tt|}tt|}n3tt|ddtt|ddd}tt|ddtt|ddd}tt|ddtt|ddd}||d|d < |S )Nr   r   r   r   r   rU   )rX   output)forward_originr   r   )r   rV   rW   retr   rh   keep_tensorsrk   r   r   _forward  s@   


zhack_forward.<locals>._forwardr   )r\   r   r~   r   )r0   rk   rh   r7   r   r   r   r   r   r]      s   )r]   c                    s`    fdd}|st |ds|jt|j|||_|_|r,t |dr.|j|_|`d S d S d S )Nc                    s     D ]-\}}tt|jddtt|jdddtt|jddtt|jdddd |< q| j|i |}  D ]\}}tt|jddtt|jddd | d< q>d ura  |S )Nr   r   r   r   )r   grad
data_after)Znamed_parametersr   r   r   r   step_origin)r   rV   rW   rk   paramr   rh   r|   r0   r   r   _step=  s8   






zhack_backward.<locals>._stepr  )r\   stepr~   r   r  )r0   rq   rh   r7   r|   r  r   r  r   r   7  s   "r   c                 C   sJ   |   D ]\}} |d ur|d | n|}t| ||| t| ||| qd S )N.)ru   r]   r^   )r0   rh   parent_namer7   rk   Z	full_namer   r   r   r^   g  s
   r^   c                 K   s  |d u rdd }t |  }t | }|| }|| }td| d|  ||}d}	|D ]}
| |
 }||
 }t|d }t|d }||||
d}|d ur`td|
 d| d	 |	o^|}	n(td
|
 d|d |d fi |os|	}	td
|
 d|d |d fi |o|	}	t|d }t|d }||||
d}|d urtd|
 d| d	 |	o|}	q/td
|
 df||d|o|	}	q/|	S )Nc                  _   r   r:   r   rU   r   r   r   rg   t  r   z(compare_io_and_print.<locals>.compare_fnzunmatched keys: , TrX   z	input of + compared with user compare_fn with result:
unmatched module z input argsrV   z input kwargsrW   r   z
output of z outputsZarg1Zarg2)setkeysr   intersectionr   r   )r   rh   rg   rW   keys1keys2addedremovedshared_keysmatchrP   v1v2Zv1inputZv2inputresv1outputv2outputr   r   r   rf   q  sn   



rf   c                 K   st  |d u rdd }t |  }t | }|| }|| }	td| d|	  ||}
d}|
D ]}|d ur:||v r:q/|| | || |d}|d urYtd| d| d	 |oW|}q/| | d
 | | d | | d }}}|| d
 || d || d }}}td| df||d|o|}|dkrtd| df||d|o|}td| d||fi |o|}q/|S )Nc                  _   r   r:   r   rU   r   r   r   rg     r   z.compare_backward_and_print.<locals>.compare_fnzunmatched backward keys: r	  Tr   zbackward data of r
  r  r   r   r  r  z tensor datar  rp   z
 grad dataz data after step)r  r  r   r  r   )r   r   r   r   rg   rW   r  r  r  r  r  r  rP   r  Zdata1Zgrad1Zdata_after1Zdata2Zgrad2Zdata_after2r   r   r   r     sz   








r   c                 K   s  |d u rdd }| d | d | d | d f\}}}}|d |d |d | d f\}}	}
}d}|||d d}|d urFt d| d	 |oD|}n6|d
 |d
 kr\t d|d
  d|d
   td|d |d fi |ok|}td|d |d fi |o{|}|||	d d}|d urt d| d	 |o|}n&|d
 |	d
 krt d|d
  d|	d
   td|d |	d fi |o|}|||
d d}|d urt d| d	 |o|}nt	d||
d|o|}|||d d}|d urt d| d	 |o|}|S td||fi |o|}|S )Nc                  _   r   r:   r   rU   r   r   r   rg     r   z.compare_cfg_and_optimizers.<locals>.compare_fnrq   rr   r   r}   Tz4optimizer compared with user compare_fn with result:r  r~   zOptimizer type not equal:z and zunmatched optimizer defaultsr   zunmatched optimizer state_dictr   z7lr_scheduler compared with user compare_fn with result:z!unmatched lr_scheduler state_dictz.cfg compared with user compare_fn with result:unmatched cfgr  z7random state compared with user compare_fn with result:zunmatched random state)r  )r   r   )r   Zcfg_jsonrg   rW   Z
optimizer1Zlr_scheduler1Zcfg1Zstate1Z
optimizer2Zlr_scheduler2Zcfg2Zstate2r  r  r   r   r   r     s   




r   c                   @   r9   )IgnoreKeyFnc                 C   s.   t |tr|g}t |tr|| _d S g | _d S r:   )r;   r   r   r  )r   r  r   r   r   r     s   
zIgnoreKeyFn.__init__c                 C   s2   | j D ]}t|}|d ur||r dS qd S r   )r  recompile	fullmatch)r   r  r  rP   r~   _keypatternr   r   r   __call__  s   

zIgnoreKeyFn.__call__N)rI   rJ   rK   r   r#  r   r   r   r   r    s    r  )Nr   )FF)FN)NFr:   )NN))r   r(   r   r_   r   r  r   rb   collectionsr   collections.abcr   pathlibr   typesr   typingr   r   r   rA   r   r<   rv   Ztorch.optimr	   Z
test_utilsr   r   r   r[   r   r   r   r]   r   r^   rf   r   r   r  r   r   r   r   <module>   sT     <t
:2

4
4
B