o
    pi                     @  s  U d dl mZ d dlZd dlmZmZmZ d dlZd dlmZm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZ ddlmZ er}d dl m!Z!m"Z" d dlm#Z# d dl$m%Z%m&Z& ed Z'de(d< g dZ)ee*ej+ddZ,G dd deZ-G dd deZ.G dd deZ/G dd deZ0e0Z1G d d! d!eZ2G d"d# d#eZ3G d$d% d%eZ4G d&d' d'eZ5G d(d) d)eZ6G d*d+ d+eZ7G d,d- d-eZ8G d.d/ d/eZ9d0d1 Z:dS )2    )annotationsN)TYPE_CHECKINGAnyLiteral)_C_ops_legacy_C_opsin_dynamic_mode)check_variable_and_dtype)_create_tensor)
get_logger)	ParamAttrcore)
functional)Constant)FakeQuantActLSQPlusFakeQuantWeightLSQPlus)unique_name   )Layer)Never	TypeAlias)Tensor)	DTypeLikeSize2)abs_maxmoving_average_abs_maxchannel_wise_abs_max
lsq_weightchannel_wise_lsq_weightlsq_actr   
_QuantType)FakeQuantAbsMaxFakeQuantMovingAverageAbsMaxFakeQuantChannelWiseAbsMaxQuantizedConv2DQuantizedConv2DTransposeQuantizedLinearMovingAverageAbsMaxScaleMAOutputScaleLayerFakeQuantMAOutputScaleLayer	QuantStubQuantizedRowParallelLinearQuantizedColumnParallelLinearQuantizedMatmulz&%(asctime)s-%(levelname)s: %(message)s)fmtc                      8   e Zd ZdZ					dd fddZdddZ  ZS )r!   a  
    FakeQuantAbsMax layer does the abs_max quant and then dequant.
    Its computational formula is described as below:

    :math:`scale = max(abs(X))`
    :math:`range = 2^{bit\_length - 1} - 1`
    :math:`Out = round(X / scale * range) * scale / range`
    N   float32Fname
str | None
quant_bitsintdtyper   quant_on_weightboolreduce_typeLiteral['max'] | NonereturnNonec                   s~   t    || _|| _|| _|r| dnd}t|| _|r:t| jt	ddd}| j
dg|| jd| _d| j_d S d | _d S )	N.scalequant_dequant.scaleMbP?Fr2   ZinitializerZ	trainable   shapeattrr6   T)super__init___quant_bits_name_reduce_typer   generate_scale_namer   r   create_parameter_dtype_scalestop_gradient)selfr2   r4   r6   r7   r9   scale_prefix
scale_attr	__class__ c/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/paddle/nn/quant/quant_layers.pyrF   O   s"   


zFakeQuantAbsMax.__init__inputr   c           	      C  sH  t  rWd| jf}t|j|j d|j|jdd}| j}| jdkr+t	j
j|t	j
jjd |s?ttjjj| jdg| jdd}d|_t|| jd\}}t|| t|| |S t|d	d
gd d| ji}d|gi}| jj|j d|jtjjjddd}| j}|s| jj| j| jtjjjddd}|g|gd}| jjd|||d |S )N
bit_length.quantized.dequantizedFtyper2   rC   r6   persistablemaxoprA   TrW   r1   r!   Xr2   r6   r[   r\   rO   ZOutZOutScale fake_quantize_dequantize_abs_maxr[   inputsoutputsattrs)r   rG   r
   r[   r2   rC   r6   rN   rI   paddledistributed
all_reduceReduceOpMAXr   VarDescVarTypeDENSE_TENSORrK   rM   rO   r   rc   assign_out_r	   _helpercreate_variable	append_op)	rP   rW   rg   	quant_out	out_scaleout1out2re   rf   rU   rU   rV   forwardj   sv   






zFakeQuantAbsMax.forward)Nr0   r1   FN)r2   r3   r4   r5   r6   r   r7   r8   r9   r:   r;   r<   rW   r   r;   r   __name__
__module____qualname____doc__rF   rx   __classcell__rU   rU   rS   rV   r!   E   s    r!   c                      r/   )r"   aM  
    FakeQuantMovingAverageAbsMax layer does the moving_average_abs_max quant and then dequant.
    Its computational formula is described as below:

    :math:`scale = (moving\_rate*accum+max(abs(x)))/(moving\_rate*state+1)`
    :math:`range = 2^{bit\_length - 1} - 1`
    :math:`Out = round(X / scale * range) * scale / range`
    N?r0   r1   r2   r3   moving_ratefloatr4   r5   r6   r   r9   r:   r;   r<   c                   s   t    || _|| _|| _|r| dnd}tt|tddd}| j	dg||d| _
d| j
_|r8| d	nd
}tt|tddd}	| j	dg|	|d| _d| j_|r[| dnd}
tt|
tddd}| j	dg||d| _d| j_d S )Nr=   r>   r?   Fr@   rA   rB   T.statezquant_dequant.state.accumzquant_dequant.accum)rE   rF   _moving_raterG   rI   r   r   rJ   r   rL   rN   rO   _state_accum)rP   r2   r   r4   r6   r9   rQ   rR   state_prefix
state_attraccum_prefix
accum_attrrS   rU   rV   rF      sD   
z%FakeQuantMovingAverageAbsMax.__init__rW   r   c              
   C  s  t  rxd| jd| jd| j f}t|j|j d|j|jdd}| j	dkr0t
jj| jt
jjjd | jr6| jnd }| jr>| jnd }t|| j||| j| j| j d	\}}}}	t|| | rft|| j |rnt|| |rvt|	| |S t|d
dgd | j| j| j d}|g| jgd}
| jj|j d|jtjjjddd}|g| jgd}| jr| jg|
d< | jg|
d< | jg|d< | jg|d< | jjd|
||d |S )Nr   rX   is_testrY   FrZ   r]   r^   rA   rW   r1   r"   )r   rX   r   )r`   ZInScalera   rb   InStateInAccumOutStateOutAccum/fake_quantize_dequantize_moving_average_abs_maxrd   )r   r   rG   trainingr
   r[   r2   rC   r6   rI   rh   ri   rj   rN   rk   rl   r   r   r   r   rp   Z_is_initializedr	   rq   rr   r   rm   rn   ro   rs   )rP   rW   rg   rt   stateaccumrv   rw   Zout3Zout4re   rf   rU   rU   rV   rx      s   



z$FakeQuantMovingAverageAbsMax.forward)Nr   r0   r1   N)r2   r3   r   r   r4   r5   r6   r   r9   r:   r;   r<   ry   rz   rU   rU   rS   rV   r"      s    -r"   c                      s8   e Zd Z							dd fddZdddZ  ZS )r#   Nr0   r   r1   Fr2   r3   channel_num
int | Noner4   r5   
quant_axisr6   r   r7   r8   r9   r:   r;   r<   c           
        s   |sJ dt    || _|| _|| _|| _|| _|| _|r$| dnd}t	|| _
|rJt| j
tddd}	| j| jg|	| jd| _d| j_d S d | _d S )	Nz5Channel_wise only can be used on weight quantization.r=   r>   g        Fr@   rB   T)rE   rF   rG   _quant_axisrM   rH   _channel_numrI   r   rJ   rK   r   r   rL   rN   rO   )
rP   r2   r   r4   r   r6   r7   r9   rQ   rR   rS   rU   rV   rF   7  s.   


z#FakeQuantChannelWiseAbsMax.__init__rW   r   c           	      C  s^  t  r_d| jd| jf}t|j|j d|j|jdd}| j}| j	dkr.t
jj|t
jjjd |d u rEttjjj| j| jg| jdd}d|_t|| jd	| j\}}t|| t|| |S t|d
dgd | jd	| jd}d|gi}| jj|j d|jtjjjddd}| j}|s| jj| j| jtjjjddd}|g|gd}| jjd|||d |S )NrX   r   rY   FrZ   r]   r^   TrA   rW   r1   r#   )rX   Z
round_typer   r`   ra   rb   -fake_channel_wise_quantize_dequantize_abs_maxrd   )r   rG   r   r
   r[   r2   rC   r6   rN   rI   rh   ri   rj   rk   rl   r   rm   rn   ro   rK   r   rM   rO   r   r   rp   r	   rq   rr   rs   )	rP   rW   rg   rt   ru   outscalere   rf   rU   rU   rV   rx   Z  s   





z"FakeQuantChannelWiseAbsMax.forward)NNr0   r   r1   FN)r2   r3   r   r   r4   r5   r   r5   r6   r   r7   r8   r9   r:   r;   r<   ry   r{   r|   r}   rF   rx   r   rU   rU   rS   rV   r#   6  s    #r#   c                      s2   e Zd Z				dd fddZdddZ  ZS )r'   Nr   r1   r2   r3   r   r   r6   r   r9   r:   r;   r<   c                   s   t    || _|| _|r| dnd}t|}t|tddd}| jdg||d| _	d| j	_
|r7| d	nd
}tt|tddd}	| jdg|	|d| _d| j_
|rZ| dnd}
tt|
tddd}| jdg||d| _d| j_
dS )a  
        MovingAverageMaxScale layer is used to calculating the output quantization
        scale of Layer. Its computational formula is described as below:

        :math:`scale = (moving\_rate*accum+max(abs(x)))/(moving\_rate*state+1)`
        :math:`Out = X`
        r=   zoutscale.scaler   Fr@   rA   rB   Tr   zoutscale.stater   zoutscale.accumN)rE   rF   r   rI   r   rJ   r   r   rL   rN   rO   r   r   )rP   r2   r   r6   r9   rQ   Z
scale_namerR   r   r   r   r   rS   rU   rV   rF     s@   


z!MovingAverageAbsMaxScale.__init__rW   r   c           
      C  sV  t  rSd| jd| j f}t|j|j d|j|jdd}| jdkr-t	j
j| jt	j
jjd | jr3| jnd }| jr;| jnd }tj||||| j||g|R  \}}}}|S t|dd	d
gd | j| j d}d|gi}| jj|j d|jtjjjddd}|g| jgd}	| jr| jg|d< | jg|d< | jg|	d< | jg|	d< | jjd||	|d |S )Nr   r   z.tmpFrZ   r]   r^   rW   r1   Zfloat64r'   )r   r   r`   ra   rb   r   r   r   r   moving_average_abs_max_scalerd   )r   r   r   r
   r[   r2   rC   r6   rI   rh   ri   rj   rN   rk   rl   r   r   r   r   r	   rq   rr   r   rm   rn   ro   rs   )
rP   rW   rg   rt   r   r   r   _re   rf   rU   rU   rV   rx     sp   




z MovingAverageAbsMaxScale.forward)Nr   r1   N)
r2   r3   r   r   r6   r   r9   r:   r;   r<   ry   r   rU   rU   rS   rV   r'     s    1r'   c                      sR   e Zd ZU dZded< ded< 									dd  fddZd!ddZ  ZS )"r$   z
    The computational logic of QuantizedConv2D is the same with Conv2D.
    The only difference is that its inputs are all fake quantized.
    r   weightbiasr0   r   r   Nlayerr   weight_bitsr5   activation_bitsr   r   weight_quantize_typer    activation_quantize_typeweight_pre_layerLayer | Noneact_pre_layerweight_quant_layeract_quant_layerr;   r<   c              
     s   t    |j| _|j| _|j| _|j| _| jdkr|j| _|j| _|j| _|j	| _	|j
| _
d| _|	d ur:|	 | _nt|| j	j||| jd| j	j| j | jd| _|
d urY|
 | _nt|| ||| jdd| _|d urn| nd | _|d ur{| | _d S d | _d S )Nzerosr   Tr2   r   r4   r6   r7   r   r   Fr2   r   r4   r6   r7   )rE   rF   _groups_stride_padding_padding_mode _reversed_padding_repeated_twice	_dilation_data_formatr   r   Z_conv2d_quant_axis_fake_quant_weight_get_fake_quant_typer2   rM   rC   _fake_quant_input	full_name_act_preprocess_weight_preprocessrP   r   r   r   r   r   r   r   r   r   r   rS   rU   rV   rF   )  sR   





zQuantizedConv2D.__init__rW   c              
   C  s   | j d ur
|  |}| |}| j}| jd ur| | j}| |}| jdkr6tj|| j| j| j	d}d| _
tj||| j| j
| j| j| j| j	dS )Nr   )modedata_formatr   )r   paddingstridedilationgroupsr   )r   r   r   r   r   r   Fpadr   r   r   Zconv2dr   r   r   r   )rP   rW   quant_inputr   quant_weightrU   rU   rV   rx   g  s2   





zQuantizedConv2D.forward	r0   r0   r   r   r   NNNNr   r   r   r5   r   r5   r   r   r   r    r   r    r   r   r   r   r   r   r   r   r;   r<   ry   r{   r|   r}   r~   __annotations__rF   rx   r   rU   rU   rS   rV   r$      s   
 >r$   c                      sV   e Zd ZU dZded< ded< 									d!d" fddZ	d#d$dd Z  ZS )%r%   a  

    The computational logic of QuantizedConv2DTranspose is the same with Conv2DTranspose.
    The only difference is that its inputs are all fake quantized.

    Examples:
        .. code-block:: python

            >>> import paddle
            >>> import paddle.nn as nn
            >>> from paddle.nn.quant.quant_layers import QuantizedConv2DTranspose

            >>> x_var = paddle.uniform((2, 4, 8, 8), dtype='float32', min=-1., max=1.)
            >>> conv = nn.Conv2DTranspose(4, 6, (3, 3))
            >>> conv_quantized = QuantizedConv2DTranspose(conv)
            >>> y_quantized = conv_quantized(x_var)
            >>> y_var = conv(x_var)
            >>> print(y_var.shape)
            [2, 6, 10, 10]
            >>> print(y_quantized.shape)
            [2, 6, 10, 10]

    r   r   r   r0   r   r   Nr   r   r   r5   r   r   r   r   r    r   r   r   r   r   r   r;   r<   c              
     s   t    |j| _|j| _|j| _|j| _|j| _|j| _|j	| _	|j
| _
d| _|	dur1|	 | _nt|| j	j||| jd| j	j| j | jd| _|
durP|
 | _nt|| ||| jdd| _|dure| nd| _|durr| | _dS d| _dS )z[
        Constructor.

        The arguments are the same as ImperativeQuantAware.
        rA   NTr   Fr   )rE   rF   r   r   r   output_padding_output_paddingr   r   r   r   Z_conv2d_transpose_quant_axisr   r   r2   rM   rC   r   r   r   r   r   rS   rU   rV   rF     sP   



z!QuantizedConv2DTranspose.__init__rW   output_sizeSize2 | Nonec                 C  s   | j d ur
|  |}| |}| j}| jd ur| | j}| |}|d u r*| j}nd}tj||| j| j	|| j
| j| j|| jd
S )Nr   )r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   Zconv2d_transposer   r   r   r   r   r   )rP   rW   r   r   r   r   r   rU   rU   rV   rx     s,   




z QuantizedConv2DTranspose.forwardr   r   N)rW   r   r   r   r;   r   r   rU   rU   rS   rV   r%     s    
 Ar%   c                      sZ   e Zd ZU dZded< ded< ded< 								
	
	
	
d!d" fddZd#dd Z  ZS )$r&   z
    The computational logic of QuantizedLinear is the same with Linear.
    The only difference is that its inputs are all fake quantized.
    r   r   r   strr2   r0   r   r   Nr   r   r   r5   r   r   r   r   r    r   r   r   r   r   r   r;   r<   c                   s   t    |j| _|j| _|j| _d| _|	d ur|	 | _nt|| jj||| jd| jj	| j | jdd	| _|
d ur=|
 | _
nt|| ||| jdd| _
|d urR| nd | _|d ur_| | _d S d | _d S )NrA   T)r2   r   r4   r6   r7   r   r   quant_linearFr   )rE   rF   r   r   r2   _linear_quant_axisr   r   rM   rC   r   r   r   r   r   rS   rU   rV   rF     sD   



zQuantizedLinear.__init__rW   c                 C  s^   | j d ur
|  |}| |}| j}| jd ur| | j}| |}tj||| j| jd}|S )Nxr   r   r2   )	r   r   r   r   r   r   linearr   r2   )rP   rW   r   r   r   r   rU   rU   rV   rx   B  s   




zQuantizedLinear.forwardr   r   ry   r   rU   rU   rS   rV   r&     s    
 7r&   c                      n   e Zd ZU ded< ded< ded< ded< ded	< ded
< 									d&d' fd!d"Zd(d$d%Z  ZS ))r,   r   r   r   r   r2   r8   is_mp#paddle.distributed.collective.Groupmodel_parallel_groupgather_outputr0   r   r   Nr   r   r   r5   r   r   r   r   r    r   r   r   r   r   Literal[None]r   r;   r<   c                   s   t    	 |	d u sJ d|
d u sJ d|j| _|j| _|j| _d| _|j| _|j| _|j	| _	t
|| jj||| jd| jj| j | jtj dkrLdnd d	| _t
|| ||| jdd d| _|d urg| nd | _|d urt| | _d S d | _d S )	NzHWhen quantizing ColumnParallelLinear, weight_quant_layer should be None.zEWhen quantizing ColumnParallelLinear, act_quant_layer should be None.rA   Tr]   r2   r   r4   r6   r7   r   r   r9   Fr2   r   r4   r6   r7   r9   )rE   rF   r   r   rH   r2   r   r   r   r   r   rM   rC   rh   ri   get_world_sizer   r   r   r   r   r   rS   rU   rV   rF   Z  sR   


z&QuantizedColumnParallelLinear.__init__rW   c                 C  s   | j rtjjj|| jd}n|}| jd ur| |}| |}| j}| j	d ur-| 	| j}| 
|}tj||| j| jd}| jrO| j rOtjjj|| jd}|S |}|S )Ngroupr   )r   rh   ri   
collectiveZ_c_identityr   r   r   r   r   r   r   r   r   r2   r   Z	_c_concat)rP   rW   input_parallelr   r   r   output_paralleloutputrU   rU   rV   rx     s,   




z%QuantizedColumnParallelLinear.forwardr   r   r   r   r5   r   r5   r   r   r   r    r   r    r   r   r   r   r   r   r   r   r;   r<   ry   r{   r|   r}   r   rF   rx   r   rU   rU   rS   rV   r,   R  $   
 Ar,   c                      r   ))r+   r   r   r   r   r2   r8   r   r   r   r   r0   r   r   Nr   r   r   r5   r   r   r   r   r    r   r   r   r   r   r   r   r;   r<   c                   s  t    |	d u sJ d|
d u sJ d|j| _|j| _|j| _d| _|j| _|j| _|j	| _	t
|| jj||| jd| jj| j | jtj dkrKdnd d	| _t
|| ||| jdtj dkrcdnd d| _|d uro| nd | _|d ur|| | _d S d | _d S )	NzQWhen quantizing RowParallelLinear, weight_quant_layer cannot defined by yourself.zNWhen quantizing RowParallelLinear, act_quant_layer cannot defined by yourself.rA   Tr]   r   Fr   )rE   rF   r   r   rH   r2   r   input_is_parallelr   r   r   rM   rC   rh   ri   r   r   r   r   r   r   r   rS   rU   rV   rF     sP   


z#QuantizedRowParallelLinear.__init__rW   c           	      C  s   | j s| js	|}n
tjjj|| jd}| jd ur| |}| |}| j	}| j
d ur0| 
| j	}| |}tj||| jd}| jrNtjjj|| jddd}n|}| jd ur\|| j }|S |}|S )Nr   )r   r   r2   T)r   Zuse_calc_streamZuse_model_parallel)r   r   rh   ri   r   Z_c_splitr   r   r   r   r   r   r   r   r2   Z_mp_allreducer   )	rP   rW   r   r   r   r   r   Zoutput_r   rU   rU   rV   rx     s6   




z"QuantizedRowParallelLinear.forwardr   r   ry   r   rU   rU   rS   rV   r+     r   r+   c                      sJ   e Zd ZdZ										d#d$ fddZ			d%d&d!d"Z  ZS )'r-   z
    The computational logic of QuantizedMatmul is the same with Matmul.
    The only difference is that its inputs are all fake quantized.
    Nr0   r   r   r   r   r   r5   r   r   r   r   r    r   r   r   r   r   r;   r<   c                   sz   t    |
d ur|
 | _|
 | _nt|||dd| _t|||dd| _|d ur+| nd | _|d ur8| | _d S d | _d S )NF)r   r4   r7   )rE   rF   _fake_quant_x_fake_quant_yr   _act_preprocess_x_act_preprocess_yr   rS   rU   rV   rF   *  s,   

zQuantizedMatmul.__init__Fr   r   ytranspose_xr8   transpose_yr2   r3   c           	      C  sR   | j d ur
|  |}| |}| jd ur| |}| |}t|||||}|S r   )r   r   r   r   rh   matmul)	rP   r   r   r   r   r2   Zquant_xZquant_yr   rU   rU   rV   rx   R  s   





zQuantizedMatmul.forward)
Nr0   r0   r   r   r   NNNN)r   r   r   r5   r   r5   r   r   r   r    r   r    r   r   r   r   r   r   r   r   r;   r<   )FFN)r   r   r   r   r   r8   r   r8   r2   r3   r;   r   rz   rU   rU   rS   rV   r-   $  s"    ,r-   c                      s8   e Zd ZdZ					dd fddZdddZ  ZS )r(   z
    Add MovingAverageMaxScale layer to the behind of the input layer.
    Calculate the scale (moving average abs max) for the output of the input layer.
    Nr   r1   r   r   r   r   r2   r3   r6   r   r9   r:   r;   r<   c                   s4   t    || _|du r| }t||||| _dS )z
        Construct
        N)rE   rF   _layerr   r'   _ma_output_scale)rP   r   r   r2   r6   r9   rS   rU   rV   rF   l  s   

zMAOutputScaleLayer.__init__re   r   kwargsr   c                 O  s.   | j |i |}t|tttfr|S | |S r   )r   
isinstancelisttupledictr   rP   re   r   r   rU   rU   rV   rx     s   
zMAOutputScaleLayer.forward)Nr   Nr1   N)r   r   r   r   r2   r3   r6   r   r9   r:   r;   r<   re   r   r   r   r;   r   rz   rU   rU   rS   rV   r(   f  s    r(   c                      s8   e Zd ZdZ					dd fddZdddZ  ZS )r)   zR
    Add FakeQuantMovingAverageAbsMax layer to the behind of the input layer.
    r0   r   Nr   r   r   r5   r   r   r   r2   r3   r9   r:   argsr   r   r;   r<   c           	   	     s>   t    || _td|d u r| n|||| jd|d| _d S )Nr   Fr   )rE   rF   r   r   r   rM   _fake_quant_output)	rP   r   r   r   r   r2   r9   r   r   rS   rU   rV   rF     s   
z$FakeQuantMAOutputScaleLayer.__init__re   r   r   c                 O  s8   | j |i |}t|ttfrt|dkr|S | |S )NrA   )r   r   r   r   lenr   r   rU   rU   rV   rx     s   
z#FakeQuantMAOutputScaleLayer.forward)r0   r0   r   NN)r   r   r   r5   r   r5   r   r   r2   r3   r9   r:   r   r   r   r   r;   r<   r   rz   rU   rU   rS   rV   r)     s    r)   c                 K  s  | dd | dd| dd| dd d}| dkr$| d	d
|d	< n| dkr1| dd|d< n| dkrX| d	d
|d	< | dd |d< | dd|d< |d d usWJ dn^| dkru| dd
|d< d
|d< d|d< | dd
|d< nA| dkrd} | dd
|d< d|d< | dd |d< | dd
|d< |d d usJ dn| dkr| dd
|d< | dd|d< tttttd}||  di |S )Nr2   r4   r0   r6   r1   r9   )r2   r4   r6   r9   r   r7   Fr   r   r   r   r   r   r   zHYou need to input channel_numwhen you use channel_wise_abs_max strategy.r   Zall_positiveZper_channelrA   r   r   Tr   Z	symmetric)r   r   r   r   r   rU   )getr!   r"   r#   r   r   )Z
quant_typer   Z	call_argsZfake_quant_maprU   rU   rV   r     sP   



r   );
__future__r   loggingtypingr   r   r   rh   r   r   r   Zpaddle.base.data_feederr	   Zpaddle.base.frameworkr
   Zpaddle.base.log_helperr   Zpaddle.frameworkr   r   Z	paddle.nnr   r   Zpaddle.nn.initializerr   Zpaddle.nn.quant.lsqr   r   Zpaddle.utilsr   Zlayer.layersr   Ztyping_extensionsr   r   r   Zpaddle._typingr   r   r    r   __all__r{   INFOZ_loggerr!   r"   r#   r'   r*   r$   r%   r&   r,   r+   r-   r(   r)   r   rU   rU   rU   rV   <module>   sP   	g ruf{QgkB"%