o + i@sddlmZddlZddlZddlmZddlZddlmZ ddl m Z ddlm Z ddlmZmZmZddlmZddlmZddlmZmZmZdd lmZdd lmZmZmZm Z erddd lm!Z!gZ"d6d d Z#d6ddZ$d6ddZ%da&ddZ'ddZ(d7ddZ)ddZ*GdddZ+Gd d!d!e+Z,d"d#Z-Gd$d%d%Z.Gd&d'd'e.Z/Gd(d)d)e.Z0da1d*d+Z2da3d,d-Z4Gd.d/d/e.Z5ej6d8d0d1Z7d2d3Z8d4d5Z9e.Z:e/Z;e0Z>> import paddle >>> from paddle.nn import clip >>> input = paddle.to_tensor([[2.0, 2.0], [2.0, 2.0]], dtype='float32') >>> reward = clip.clip_by_norm(x=input, max_norm=1.0) >>> print(reward) Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True, [[0.50000000, 0.50000000], [0.50000000, 0.50000000]]) clip_by_normX)float16float32uint16max_normN.tmpF)typenamedtypeZ persistableOutrinputsattrsoutputs)r)rrrrlocalsrr floatrZgenerate_with_ignorable_keyjoinrZcreate_variablerr append_op)xrrhelperoutr*Z/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/paddle/nn/clip.pyr+s**   rcCsLtrt|Stdit}|j|jd}|jdd|iid|id|S)a Merge by adding duplicated rows in the input SelectedRows object. Args: x(Tensor): The input selected rows to be merge. name(basestring|None): Name of the output. Returns: Tensor, merged output. Examples: .. code-block:: python >>> import paddle >>> import paddle.base as base >>> b = paddle.static.default_main_program().global_block() >>> var = b.create_var( ... name="X", dtype="float32", persistable=True, ... type=base.core.VarDesc.VarType.SELECTED_ROWS) >>> y = paddle.nn.clip.merge_selected_rows(var) merge_selected_rowsrrrrN)r,)rrr,rr#"create_variable_for_type_inferencerr&r'rr(r)r*r*r+r,qs r,cCsrtrt|St|dtd|jtjjj krt dt d it }|j |jd}|jdd|id|iid|S) aC Get tensor data from input with SelectedRows type, and outputs a Tensor. .. code-block:: text input x is SelectedRows: x.rows = [0, 5, 5, 4, 19] x.height = 20 x.value = [[1, 1] [2, 2] [2, 2] [3, 3] [6, 6]] Output is DenseTensor: out.shape = [5, 2] out.data = [[1, 1], [2, 2], [2, 2], [3, 3], [6, 6]] Args: x(SelectedRows): Input with SelectedRows type. The data type is float32, float64, int32 or int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` . Returns: Variable: DenseTensor transformed from SelectedRows. The data type is same with input. Examples: .. code-block:: python >>> import paddle >>> import paddle.base as base >>> from paddle.base import core >>> paddle.enable_static() >>> scope = core.Scope() >>> block = paddle.static.default_main_program().global_block() >>> x_rows = [0, 5, 5, 4, 19] >>> height = 20 >>> x = scope.var('X').get_selected_rows() >>> x.set_rows(x_rows) >>> x.set_height(height) >>> x = block.create_var(name="X", dtype="float32", persistable=True, type=base.core.VarDesc.VarType.SELECTED_ROWS) >>> z = paddle.nn.clip.get_tensor_from_selected_rows(x) r'get_tensor_from_selected_rowszGThe type of 'x' in get_tensor_from_selected_rows must be SELECTED_ROWS.r-rrrr r"r!N)r0)rrr0r r rrVarDescVarType SELECTED_ROWS TypeErrorrr#r.rr&r/r*r*r+r0s , r0FcGsBt|dksJt|dkrt|dtsJt}|da|StS)Nr)len isinstancebool'_clip_by_global_norm_using_mp_type_flagargs old_valuer*r*r+"_clip_by_global_norm_using_mp_types r>cCsd|jtjjjks|jtjjjkrtr|tjjjS|jt j ks'|jt j kr0tr0|t j S|SN) rrr2r3FP16BF16r>astypeFP32r FLOAT16BFLOAT16FLOAT32r'r*r*r+_cast_to_mp_type_if_enableds rHgradr clip_inputcCs:|r|s dS|s|rt|jdkrdSdS)NFrT)Z_is_initializedZis_distZis_denser7shape)rIrJr*r*r+_can_inplace_clip_grads rLcCspt|}tr t|Sd}t|dgd|t|fit}||j}d|i}d|i}|j |||d|S)z1 Return the squared L2 norm of a tensor. squared_l2_normr')rfloat64rrrrrr r") rHrrrMrrr#r.rr&)r'Zop_typer(r)r r"r*r*r+_squared_l2_norms   rPc@seZdZddZddZdS)BaseErrorClipAttrcCtr?NotImplementedErrorselfr*r*r+__str__zBaseErrorClipAttr.__str__cCrRr?rS)rVblock grad_namer*r*r+_append_clip_oprXz!BaseErrorClipAttr._append_clip_opN)__name__ __module__ __qualname__rWr[r*r*r*r+rQs rQc@s*eZdZdZd ddZddZddZdS) ErrorClipByValuea1 Clip tensor values to the range [min, max]. Given a tensor ``t`` (see Examples below), this operation clips its value \ to ``min`` and ``max`` inplace. - Any values less than min are set to min. - Any values greater than max are set to max. Args: max (float): The maximum value to clip by. min (float, optional): The minimum value to clip by. if not set by user, \ will be set to ``-max`` by framework. Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> BATCH_SIZE = 128 >>> CLIP_MAX = 2e-6 >>> CLIP_MIN = -1e-6 >>> prog = paddle.static.Program() >>> with paddle.static.program_guard(main_program=prog): ... image = paddle.static.data(name='x', shape=[None, 784], dtype='float32') ... hidden1 = paddle.static.nn.fc(image, size=128, activation='relu') ... hidden2 = paddle.static.nn.fc(hidden1, size=64, activation='relu') ... predict = paddle.static.nn.fc(hidden2, size=10, activation='softmax') ... label = paddle.static.data(name='y', shape=[1], dtype='int64') ... cost = paddle.nn.functional.cross_entropy(input=predict, label=label) ... avg_cost = paddle.mean(cost) >>> prog_clip = prog.clone() >>> prog_clip.block(0).var(hidden1.name)._set_error_clip( ... paddle.nn.clip.ErrorClipByValue( ... max=CLIP_MAX, min=CLIP_MIN)) NcCs0t|}|dur | }nt|}||_||_dSr?)r$maxminrVr`rar*r*r+__init__Es  zErrorClipByValue.__init__cCd|jdd|jdS)Nz ByValue, min=f, max=rar`rUr*r*r+rWNzErrorClipByValue.__str__cCsP|j}|d|d|g|d|g|d|j|d|jdS)Ncliprrrar`)descr&set_typeZ set_inputZ set_output _set_attrrar`)rVrYrZZ clip_op_descr*r*r+r[Qs  z ErrorClipByValue._append_clip_opr?)r\r]r^__doc__rcrWr[r*r*r*r+r_s  & r_cs||j|jd}fdd|DD]&}||}t|dd}|dus4t|ts4td|dur>| ||qdS)Nr6csg|]}|vr|qSr*r*).0nZ grad_to_varr*r+ ^sz'error_clip_callback.. error_clipzIVariable's error_clip should be an instance of BaseErrorClipAttr or None.) rjopZop_sizeZoutput_arg_namesZ_var_recursivegetattrr8rQr5r[)rYcontextZop_descZgrad_nZfwd_varrrr*rpr+error_clip_callbackZs  rvcs^eZdZfddZddZeddZddZd d Z dddZ ddZ ddZ Z S) ClipGradBasecstdSr?)superrcrU __class__r*r+rclzClipGradBase.__init__cCrRr?rSrUr*r*r+rWorXzClipGradBase.__str__cCrRr?rSrV params_gradsr*r*r+ _dygraph_cliprszClipGradBase._dygraph_clipcCrRr?rSr|r*r*r+ _pir_clipvrXzClipGradBase._pir_clipcCrRr?rSr|r*r*r+ _static_clipyrXzClipGradBase._static_clipr}list[tuple[Tensor, Tensor]]returncCsVtr||Str||S|D]\}}t|dddur%tdnq||S)Ngradient_clip_attrz'set_gradient_clip' will be ineffective, because you have set 'need_clip' in 'ParamAttr'. So, 'set_gradient_clip' is redundant and you can remove it.)rr~rrrtwarningswarnr)rVr}pgr*r*r+__call__|s    zClipGradBase.__call__cCrRr?rSrVruparamrIr*r*r+_process_contextrXzClipGradBase._process_contextcCrRr?rS)rVrrIr*r*r+_create_operatorsrXzClipGradBase._create_operators)r}rrr)r\r]r^rcrWimperative_baseno_gradr~rrrrr __classcell__r*r*ryr+rwks   rwcsheZdZUdZded<ded<ddfd d Zdd d ZeddZ ddZ ddZ ddZ Z S)ClipGradByValuea Limit the value of multi-dimensional Tensor :math:`X` to the range [min, max]. - Any values less than min are set to ``min``. - Any values greater than max are set to ``max``. The multi-dimensional Tensor :math:`X` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). Note: ``need_clip`` of ``ClipGradByValue`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to specify the clip scope. Args: max (float): The maximum value to clip by. min (float, optional): The minimum value to clip by. if not set by user, it will be set to ``-max`` automatically. In this case, ``max`` must be greater than :math:`0`. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByValue(min=-1, max=1) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() r$r`raN float | NonerNonecs<t|dur|dksJ| }t||_t||_dS)N)rxrcr$r`rarbryr*r+rcs   zClipGradByValue.__init__strcCrd)NzClip Gradient By Value, min = rerfrgrUr*r*r+rWrhzClipGradByValue.__str__cCsbg}|D]*\}}|dur qt|dddur|||fqtj||j|jd}|||fq|S)N need_clipTFr'rar`)rtappendpaddlerirar`rVr}params_and_gradsrrnew_gradr*r*r+r~s zClipGradByValue._dygraph_clipc Csg}i}tdT|D]I\}}|durq t|dddur%|||fq |jj||gtj||j |j d}Wdn1sCwY|||f|j ||j <q Wdn1s`wYt |||S)N gradient_cliprTFr) r name_scopertrrYprogram_optimized_guardrrirar`r_correct_clip_op_role_varrVr}rparam_new_grad_name_dictrrrr*r*r+rs$   zClipGradByValue._static_clipcCdSr?r*rr*r*r+rrXz ClipGradByValue._process_contextcCstj||j|jd}||fS)Nr)rrirar`rVrrIrr*r*r+rsz!ClipGradByValue._create_operatorsr?)r`r$rarrrrr)r\r]r^rm__annotations__rcrWrrr~rrrrr*r*ryr+rs '   rcsneZdZUdZded<dfdd Zdd d Zd d Ze d dZ ddZ ddZ ddZ ddZZS)ClipGradByNorma Limit the l2 norm of multi-dimensional Tensor :math:`X` to ``clip_norm`` . - If the l2 norm of :math:`X` is greater than ``clip_norm`` , :math:`X` will be compressed by a ratio. - If the l2 norm of :math:`X` is less than or equal to ``clip_norm`` , nothing will be done. The multidimensional Tensor :math:`X` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). The clipping formula is: .. math:: Out = \left\{ \begin{array}{ccl} X & & if (norm(X) \leq clip\_norm) \\ \frac{clip\_norm*X}{norm(X)} & & if (norm(X) > clip\_norm) \\ \end{array} \right. where :math:`norm(X)` represents the L2 norm of :math:`X`. .. math:: norm(X) = ( \sum_{i=1}^{n}|x\_i|^2)^{ \frac{1}{2}} Note: ``need_clip`` of ``ClipGradByNorm`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to specify the clip scope. Args: clip_norm(float): The maximum norm value. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByNorm(clip_norm=1.0) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() r$ clip_normrrcstt||_dSr?)rxrcr$r)rVrryr*r+rc+s zClipGradByNorm.__init__rcCd|jdS)Nz!Gradient Clip By Norm, clip_norm=rerrUr*r*r+rW/r{zClipGradByNorm.__str__cCs\g}|D]'\}}|dur qt|dddur|||fqt||jd}|||fq|S)NrTFr'r)rtrrrrr*r*r+_clip_gradients2s zClipGradByNorm._clip_gradientscC ||Sr?rr|r*r*r+r~>s zClipGradByNorm._dygraph_clipcCrr?rr|r*r*r+rBs zClipGradByNorm._pir_clipc Csg}tdSi}|D]F\}}|durq t|dddur%|||fq |jj||gt||jd}Wdn1s@wY|j ||j <|||fq Wdn1s]wYt |||S)NrrTFr) rrrtrrYrrrrrrrr*r*r+rEs$    zClipGradByNorm._static_clipcCrr?r*rr*r*r+rWrXzClipGradByNorm._process_contextcCst||jd}||fS)Nr)rrrr*r*r+rZsz ClipGradByNorm._create_operators)rr$rrr)r\r]r^rmrrcrWrrrr~rrrrrr*r*ryr+rs 6  rcG>t|dkrtSt|dkrt|dtsJt}|da|SNrr6)r7&_allow_pure_fp16_global_norm_clip_flagr8r9r;r*r*r+!_allow_pure_fp16_global_norm_clipb rcGrr)r7&_allow_pure_bf16_global_norm_clip_flagr8r9r;r*r*r+!_allow_pure_bf16_global_norm_clipprrcs|eZdZUdZded<ded<ded<  ddfd d ZdddZeddZ ddZ ddZ ddZ ddZ ZS)ClipGradByGlobalNorman Given a list of Tensor :math:`t\_list` , calculate the global norm for the elements of all tensors in :math:`t\_list` , and limit it to ``clip_norm`` . - If the global norm is greater than ``clip_norm`` , all elements of :math:`t\_list` will be compressed by a ratio. - If the global norm is less than or equal to ``clip_norm`` , nothing will be done. The list of Tensor :math:`t\_list` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). The clipping formula is: .. math:: t\_list[i] = t\_list[i] * \frac{clip\_norm}{\max(global\_norm, clip\_norm)} where: .. math:: global\_norm = \sqrt{\sum_{i=0}^{N-1}(l2norm(t\_list[i]))^2} Note: ``need_clip`` of ``ClipGradyGlobalNorm`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to specify the clip scope. Args: clip_norm (float): The maximum norm value. group_name (str, optional): The group name for this clip. Default value is ``default_group``. auto_skip_clip (bool, optional): skip clipping gradient. Default value is ``False``. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() r$rr group_namer9auto_skip_clip default_groupFrrcs>tt||_||_t|tsJ||_d|_d|_ dS)NF) rxrcr$rrr8r9r _async_add_nshould_comm_on_shard_dim)rVrrrryr*r+rcs   zClipGradByGlobalNorm.__init__cCr)Nz)Gradient Clip By GlobalNorm, global_norm=rerrUr*r*r+rWr{zClipGradByGlobalNorm.__str__cCsg}g}g}g}d}t|dkr t|ddkr |ddj}nd}|D]\}} | dur-q$t|dddur6q$| } trH| rHt| } | } n| jtj j j krXt| } t | } t | } |dur| j|krd}t| j} t| jjt| jkrd}t| | | j} t| || j} | jtjks| jtjkr|| q$| jtjkr|| q$|| q$t|t|t|dkr|Sdd} t|dkrdnd}g}t|dkr| |}|||t|dkr| |}|dkr||n|||t|dkr | |}||| |}|rK|durKt}|rKd |jvrK|d d krK|d d }| }tj!|tj"j#|d t$||j|j}|j%rbt&|d rbtj'j!| |j(d )|j%ryt&|drytj'j!| |j*d )t+|}tj,d g||j-d}d}|j.sd}tj/|tj0||dd}n||krd}tj/||d}|D]\}} | durqt|dddur||| fq|rV|j| jkr|| jn|}|j| jkr3t| jjt|jjkr|j}d}|D] }|1sd}nq|r | }n't2dt| j} t| jjt| jkr)t|| |j}tj'|| j|j}t3| |rG| 4|||| fqt5| |}|||fq||| fq|S)NTrrFcSt|Sr?rstacksumZvar_listr*r*r+ async_add_n r{z7ClipGradByGlobalNorm._dygraph_clip..async_add_nrNrppr6)rsgroupsharding_group)rmp_grouprKrZ fill_valuer'yzLReshard a sharded tensor from a local mesh to a global mesh is not supported)6r7 process_meshrtrZis_selected_rowsr,Z_get_tensor_from_selected_rowsrrr2r3r4r0rPr set process_idsdistreshard placementsrrrbfloat16rrrBZget_meshZ dim_namesZ get_dim_sizeZget_submesh_with_dimZ get_groupZ _local_value all_reduceReduceOpSUMZ shard_tensorrhasattr distributedrwaitrsqrtfullrrdividemaximumZ is_replicatedrTrLZ multiply_multiply)rVr}rsum_square_listsum_square_list_fp16sum_square_list_fp32Zflag_auto_hybrid_ppZsrc_meshrr merge_grad sum_squareZpp_meshr sum_dtypeglobal_norm_varglobal_norm_var_fp16global_norm_var_fp32global_norm_var_fp64Zg_meshZpp_groupZglobal_norm_var_localmax_global_normrclip_varrJrZ is_replicateZ placementrr*r*r+r~s&                                 z"ClipGradByGlobalNorm._dygraph_clipc% Csg}g}g}g}g}g}g}g} g} g} d} t} d}|D]?\}}|r\| |jd|jjvr\|dur?|j}q|j}t|jt|jkrP|}t|jt|jks\Jqt| dkrqddlm}d} |dusqJ|D]\}}|dur|qst |dddurqs|}t r| rt |}t |}t|}| r|j|krtj||||j|j|j}|jr|jdr|jtjks|jtjkr||qs|jtjkr||qs||qs|jr |jtjks|jtjkr||qs|jtjkr||qs||qs|jtjks|jtjkr!| |qs|jtjkr.| |qs| |qst|t|t|t|t|t|t| t| t| dkr]|Sdd }t|t|t| dkrrd nd }g}g}g}t|dkr||}|||t|dkr||}|||t| dkr|| }|||t|dkr||}|d kr||n|||t|dkr||}|d kr||n|||t| dkr || }|d kr||n|||t|dkr||}||t|dkr-||}||t| dkr=||}||d}t|dkrJ||}t|dkrV||}n|jrg|jrgtjdg|d d }|jr|jrtj !||j"j#t$j%j&}tj !||j'j#t$j%j&}|dur|}n||}t|dkr||}n|jr|j(rtjdg|d d }|jr|j(rtj !||j"j#t$j%j&}|dur|}n||}t)|}tjdg|j|j*d } d}!|j+sd}!tj,| tj-|| dd}"n|| krd}!tj,| |d}"|D]l\}}|durqt |dddur"|||fq|!ri|"j|jkr2|"|jn|"}#| rZ|#j|jkrZtj|#|j||#j|#j|#j}#t.||#}$|||$fq|||fq|S)NFrr6) to_placementsTrZ no_fusioncSrr?rrr*r*r+rr{z3ClipGradByGlobalNorm._pir_clip..async_add_nrNrrrr)/rZis_dist_dense_tensor_typeaddZ dist_attrrrr7Z/paddle.distributed.auto_parallel.placement_typerrtris_selected_row_typer,r0rPrrrZ dims_mappingZ partial_dimsrZ optimize_attrrr rDrErrFZis_distributedrBZhas_dist_paramrrrridrrrrZhas_not_dist_paramrrrrrr)%rVr}rZno_fusion_sum_squareZno_fusion_sum_square_fp16Zno_fusion_sum_square_fp32Zsum_square_distZsum_square_dist_fp16Zsum_square_dist_fp32Zsum_square_not_distZsum_square_not_dist_fp16Zsum_square_not_dist_fp32Zauto_parallel_ppZ pp_meshesZpp_stage0_meshrrZp_meshrrrrrZno_fusion_global_normZglobal_norm_distZglobal_norm_not_distrrrrZglobal_norm_dist_varZglobal_norm_not_dist_varrrrrJrr*r*r+rs                                                zClipGradByGlobalNorm._pir_clipc sg}g}g}g}g}fdd}td|D]p\}} | dur"qt|dddur+q| } |jj|| gK| jtjj j krGt | } t | } t | } | jtjj jkrY|| n!| jtjj jkrg|| n| jtjj jkru|| n|| Wdn1swYqt|dkrt|dkrtdt|t|t|dkrt|t|t|dkr|WdS|jj|| gt|dkrd nd } g} t|dkr||}|s|sts| || n| |t|dkr||}|s |s ts| || n| |t|dkr5||}| d kr-| |n| || t|dkrE||}| |t| d krP|| n| d} tj| d } tjd g| jjd }tj|tj|| dd}Wdn 1s|wYi}|D]\}} | durqt|dddur||| fq|jj|| ggt| }|jtjj jkr|jtjj jkr|d}n|jtjj jkr|jtjj jkr|d}n|}t }|j!d||dd|id|| ur |j!dd|id| i|j| jddWdn 1swY| j"||j"<||| fqWdn 1s5wYt#|||S)Ncsjr t|St|Sr?)rrrrZadd_nrrUr*r+_add_ns z1ClipGradByGlobalNorm._static_clip.._add_nrrTFrz1FP16 and BF16 are not supported at the same time.rNrr6rGrrrrelementwise_mulrYrrOcastr)Zin_dtypeZ out_dtyper1)$rrrtrYrrrrr2r3r4r,r0rPrr@rrArCr7rTrrBrrrrrrrrHr Z current_blockr&rr)rVr}rrrZsum_square_list_bf16rrrrrrrrrZglobal_norm_var_bf16rZglobal_norm_var_other_dtyperZ scale_varrZnew_gZ scale_inputrYr*rUr+rs      (        8       $ z!ClipGradByGlobalNorm._static_clipcCs|j|vr#g||j<|j||jd<tjdg|j|jd||jd<n|j||jdks1td|}|jtjj j krDt |}t |}nt rS|rSt |}t |}t|}||j|||_dS)NZ _clip_valuer6r_clipz>All parameters' 'clip_norm' of a same group should be the same)rrrrr ValueErrorrrr2r3r4r,r0rrrPrru)rVrurrIrZlocal_norm_varr*r*r+r:s(     z%ClipGradByGlobalNorm._process_contextcCsdd}|jd}||jvr<||j|j}tj|d}|j|jd}tj|tj||dd}|jdks7J||j|<trLt||j|}||fS|j j d||j|d d |id ||fS) NcSrr?rrr*r*r+rUr{z;ClipGradByGlobalNorm._create_operators..async_add_nZ_scalerGrr)r6rrrrO) rrurrrrrKrrrYr&)rVrrIrZgroup_scale_nameZgroup_norm_varrZgroup_scale_varr*r*r+rTs*     z&ClipGradByGlobalNorm._create_operators)rF)rr$rrrr9rrr)r\r]r^rmrrcrWrrr~rrrrrr*r*ryr+r{s& 4  F  rcstdt|tstddurtdjD]}d| vr2d| dvr2tdnq|dur>d }t dd |DrPfd d |D}t d d |Ds]td |D]}t ||_q_dS)a Warning: This API must be used after building network, and before ``minimize`` , and it may be removed in future releases, so it is not recommended. It is recommended to set ``grad_clip`` when initializing the ``optimizer`` , this is a better method to clip gradient. There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . To specify parameters that require gradient clip. Args: grad_clip (GradientClipBase, optional): Gradient clipping strategy, it's an instance of some derived class of ``GradientClipBase`` . There are three clipping strategies ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` ). Default value: None, and there is no gradient clipping. param_list (list(Variable), optional): Parameters that require gradient clip. It can be a list of parameter or a list of parameter's name. Default None, meaning that all parameters in the program will be included. program (Program, optional): The program where parameters are located. Default None, meaning that using :ref:`api_paddle_static_default_main_program` . Returns: None Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> def network(): ... image = paddle.static.data(name='image', shape=[ ... None, 28], dtype='float32') ... param_attr1 = paddle.ParamAttr("fc1_param") ... fc1 = paddle.static.nn.fc(image, size=10, weight_attr=param_attr1) ... param_attr2 = paddle.ParamAttr("fc2_param") ... fc2 = paddle.static.nn.fc(fc1, size=10, weight_attr=param_attr2) ... loss = paddle.mean(fc2) ... return loss >>> # network 1: clip all parameter gradient >>> with paddle.static.program_guard(paddle.static.Program(), paddle.static.Program()): ... loss = network() ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByGlobalNorm(clip_norm=2.0)) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 2: clip parameter gradient by name >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByValue(min=-1.0, max=1.0), ... param_list=["fc1_param", "fc2_param"]) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 3: clip parameter gradient by value >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... param_var1 = paddle.static.default_main_program().global_block().var("fc1_param") ... param_var2 = paddle.static.default_main_program().global_block().var("fc2_param") ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByValue(min=-1.0, max=1.0), ... param_list=[param_var1, param_var2]) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 4: use 'set_gradient_clip' and 'optimize(grad_clip=clip)' together >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... clip1 = paddle.nn.ClipGradByValue(min=-1.0, max=1.0) ... clip2 = paddle.nn.ClipGradByNorm(clip_norm=1.0) ... # Set the gradient clipping strategy: clip1 ... paddle.nn.clip.set_gradient_clip(clip1) ... # Set the gradient clipping strategy: clip2 ... sgd = paddle.optimizer.SGD(learning_rate=1e-3, grad_clip=clip2) ... sgd.minimize(loss) ... # 'set_gradient_clip' will not take effect when setting has a conflict, ... # and the gradient clipping strategy will be 'clip2' zCaution! 'set_gradient_clip' is not recommended and may be deprecated in future! We recommend a new strategy: set 'grad_clip' when initializing the 'optimizer'. This method can reduce the mistakes, please refer to documentation of 'optimizer'.z<'clip' should be an instance of ClipGradBase's derived classNr op_namescopeZ optimizerz'minimize' has been invoked before, this will make 'set_gradient_clip' be ineffective! Please invoke 'set_gradient_clip' before 'minimize'.css|]}t|tVqdSr?)r8rrnelemr*r*r+ sz$set_gradient_clip..csg|] }d|qS)r)rYvarrrr*r+rqsz%set_gradient_clip..css|] }t|tjVqdSr?)r8r Parameterrr*r*r+rszK'param_list' should be a list of Parameter or basestring(parameter's name).)rrr8rwr5rr rYopsZ all_attrsattrZall_parametersallcopydeepcopyr)riZ param_listrrsrr*rr+set_gradient_cliprs:Z rc Csi}|D]d\}}|dur q|jj||gJtd5t|dd}|dur9|WdWdSt|tsBtd|j |||dWdn1sTwYWdn1scwYqg}i}|D]M\}}|durxqo|jj||g3td|j ||d\}}|j ||j <| ||gWdn1swYWdn1swYqot |||S)Nrrz8clip attribute should be an instance of GradientClipBase)rurrI)rrI)rYrrrrrtr8rwr5rrrrr) Z param_gradsrurrZ clip_attrresrrrr*r*r+append_gradient_clip_opssN  "    rc Csg}t|dkr dS|D]I\}}|durq |jj}||vrq |||jjjD]*}|drTd|dvrT|drT|dd}||vrT|||g}| d|q*q dS)NrrrZ op_role_var) r7rYidxrrZ global_blockrZhas_attrrrl) r}rZ block_id_listrrIZblock_idrs param_nameZ correct_p_gr*r*r+rs2    rr?)rIrrJr)NN)> __future__rrrtypingrrZpaddle.autogradZautogradrZpaddle.distributedrrrZ paddle.baserrrZpaddle.base.data_feederrZpaddle.base.libpaddler Zpaddle.common_ops_importr r r Z"paddle.distributed.utils.moe_utilsr Zpaddle.frameworkrrrrr__all__rr,r0r:r>rHrLrPrQr_rvrwrrrrrrrZdygraph_not_supportrrrZGradientClipBaseZGradientClipByValueZGradientClipByNormZGradientClipByGlobalNormr*r*r*r+sd           F &?  <*]m  | '