o + i)@stddlmZddlZddlZddlZddlZddlZddlZddlm Z ddlm Z m Z m Z m Z ddlZddlmZddlZddlmZmZmZddlmZddlmZmZmZdd lmZdd lmZdd l m!Z!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(m)Z)ddl*m+Z+m,Z,m-Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6ddl7m8Z8m9Z9ddl:m;Z;ddlm?Z?e rddl@mAZAmBZBddlCmDZDmEZEmFZFmGZGddlHmIZIgZJe degefZKe deegefZLe eMeNefej eNeffZOe eOgdfZPeQdZReQdZSddZTddZUddZVd d!ZWd"d#ZXd$d%ZYGd&d'd'e6ZZGd(d)d)Z[Gd*d+d+Z\Gd,ddZ]dS)-) annotationsN) OrderedDict) TYPE_CHECKINGAnyCallableUnion)Self)Tensornnprofiler)ValueSet)core framework unique_name)VarDesc)no_grad)_convert_into_variablein_declarative_modein_sot_simulation_modein_to_static_mode)_append_activation_in_dygraph)Executor global_scope) ParameterProgram_current_expected_placeconvert_np_dtype_to_dtype_default_main_programin_dygraph_mode in_pir_mode name_structpaddle_type_to_proto_type)LayerHelperBase)ShardedStateDictbuild_sharded_state_dict) ParamAttr)in_profiler_mode) deprecated)IterableSequence) DTypeLike ParamAttrLike PlaceLike ShapeLike) InitializerLayerz(.)([A-Z][a-z]+)z([a-z])([A-Z])cCsTts(|jjdkrttj|j_d|j_dSd|j_t |j ddSdS)zC A pre-hook to mark op numbers before enter layer.forward. rTFzX has recorded the op information before. Please check whether you call this layer twice.N) r _op_recorderstartlenr current_blockopsis_validwarningswarn _full_name)layerinputsr;b/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/paddle/nn/layer/layers.pyrecord_program_ops_pre_hookUs    r=c CstsO|jjrQ|jj}ttj}|dkr||ksJtj||}||j_||j_|D]}|j D] \}}| ||q8q1|jj D] } | qHdSdSdS)zd A post-hook to append customized attributes into all operators generated in current layer. rN)rr0r5r1r2rr3r4end_customized_attrsitemsZ _set_attrhooksremove) r9r:outputsr1r>r4op attr_namevalZ hook_helperr;r;r<!set_op_customized_attrs_post_hookfs  rGcCsdddd}|||S)NZlinearZ embedding)Zrow_parallel_linearZcolumn_parallel_linearZvocab_parallel_embedding)get)Z dist_scopemappingr;r;r<_scope_dist2single|s  rJcCstd|}td|S)Nz\1_\2) _first_cap_resub _all_cap_relower)names1r;r;r<_convert_camel_to_snakes rQcCsd|d}t|dkr |Sg}t|D]\}}|dkr&|t|d|q|ddd|S)N r )splitr2 enumerateappendstrjoin)stringindentrPs2idxliner;r;r< _addindents  r_cCs$t||vrdS|j|ddddS)NTF)dtype floating_onlyinclude_sublayers)type_to_impl)r9r`excluded_layersr;r;r<_layer_trans_dtypes rfcsveZdZfddZ     dddZddZdd Zdd d Zdd d ZddZ ddZ dddZ ddZ Z S)LayerObjectHelpercstj||ddS)N) layer_type)super__init__selfrO __class__r;r<rjszLayerObjectHelper.__init__NcCs|jj|||||dS)aappend an operator for this layer object. Args: type: operator type inputs: input variable of the operator dtype: data type of this parameter is_bias: if this is a bias parameter default_initializer: set the default initializer for this parameter Returns created parameter Variable. )rcr:rCattrs stop_gradient) main_programr3 append_op)rlrcr:rCrorpr;r;r<rrs zLayerObjectHelper.append_opcCsH|}g}t|ttfr|D] }|||q |S||||SN) isinstancelisttuplerWZ to_variable)rl inputs_inr:retinpr;r;r<_multiple_inputsz!LayerObjectHelper._multiple_inputcCs*||}t|dkr|jd|dS)NrSz layer only takes one input inr)rzr2rh)rlrwr:r;r;r<_inputs   zLayerObjectHelper._inputcCs|}t|tr |g}t|dkrt||krtd|jt|dkr?|dkr?dg|}t|D] }t|d||<q1|}|S)NrSzparameter number mismatch in r)rtr%r2 ValueErrorrOrangecopydeepcopy)rllength param_attr_inZ param_attrtmpir;r;r<_multiple_param_attrs   z&LayerObjectHelper._multiple_param_attrccsdt|}t|trtd|j|dur|ng}||}|t||}t ||EdHdS)zAccess all inputs and params one by one Args: inputs_in: inputs to be iter param_attr_in: param_attr to be iter Returns input, param_attr z"Param_attr should not be False in N) r%Z_to_attrrtboolr|rOrzrr2zip)rlrwrr:Z param_attrsr;r;r<iter_inputs_and_paramss  z(LayerObjectHelper.iter_inputs_and_paramscCsd|dur|ng}||}d}|D]}|dur|j}q||jkr/td|d|jd|jq|S)zGet input data type Args: inputs_in: inputs wanted know the data type Returns dtype of the input NzData Type mismatch: z to z in )rzr`r|rO)rlrwr:r`Zeachr;r;r< input_dtypes  zLayerObjectHelper.input_dtypecCs4|j|}t|tstd|d|j|S)z}Get parameter specifically Args: name: parameter's name Returns target parameter zno Parameter name z found in )rq global_blockvarrtrr|rO)rlrOparamr;r;r< get_parameters zLayerObjectHelper.get_parametercCs|}|dur|St|trd|i}n t|d|j|dur&|r&||d<|d}tr6t|||}|S|j|jd}|j |d|gid|gi|d|S) a%Append activation Args: input_var: the input variable. The len(input_var.shape) is larger or equal than 2. act: activation type use_cudnn: if use cudnn Return the Variable of after append activation Nrcz should be unicode or str in use_cudnnr`XZOut)rcr:rCro) rtrX TypeErrorrOpoprrZ"create_variable_for_type_inferencer`rr)rlZ input_varZactrZact_typeresrr;r;r<append_activations(      z#LayerObjectHelper.append_activationcCs(|}t||std||j|j|jdS)zCheck if the input parameter is instance of input class Args: param: parameter to be check cls: class of the parameter Return result of the check (True or False) z?The input {0} parameter of method {1} must be {2}, in layer {3}N)rtrrh__name__rO)rlrclsr;r;r< is_instance2s zLayerObjectHelper.is_instance)NNNNNrs)NN)r __module__ __qualname__rjrrrzr{rrrrrr __classcell__r;r;rmr<rgs      #rgc@seZdZdZdddZdS)LayerOpsRecorderz= Record generated operators information in nn.Layer. NFcCs"||_||_||_||_||_dSrs)r1r>r4r5rA)rlr1r>r4r5rAr;r;r<rjKs  zLayerOpsRecorder.__init__)rrNFN)rrr__doc__rjr;r;r;r<rFsrc@s8eZdZUdZdZded<dddd dZdddZdS)HookRemoveHelperz3A HookRemoveHelper that can be used to remove hook.rint next_hook_idNextra_hook_dictrA+typing.OrderedDict[int, Callable[..., Any]]rrreturnNonecCsDt||_tj|_tjd7_d|_|dur t||_dSdS)NrS)weakrefref _hooks_refrr_hook_id_extra_hooks_ref)rlrArr;r;r<rjXs zHookRemoveHelper.__init__cCs^|}|dur|j|vr||j=|jdur)|}|dur+|j|vr-||j=dSdSdSdSrs)rrr)rlrAZ extra_hooksr;r;r<rBfs  zHookRemoveHelper.remove)rArrrrr)rr)rrrrr__annotations__rjrBr;r;r;r<rSs  rc@seZdZUdZded< ddd d ZdddZdddZdddZdddZ dddZ dd dd#d$Z     ddd0d1Z e d2d3d4d5   ddd9d:Z   ddd;d<Zddd@dAZdddBdCZddEdFZddHdIZdddLdMZ N = =dddRdSZ N   =dddVdWZ =dddYdZZddd[d\Z N = =ddd]d^Zddd`daZddedfZddhdiZddjdkZddldmZddndoZddqdrZ ddtduZ!dvdwZ"ddydzZ#dd|d}Z$dd~dZ%dddZ&dddZ'dddZ(dddZ)dddZ*dddZ+  = NddddZ,  = N  = =ddddZ-  = N = =ddddZ.  = N = =ddddZ/ NddddZ0e1j2 =ddddZ3   ddddZ4 =ddddZ5dddZ6    = ddddZ7dddZ8e3Z9e3Z: ddddZ; ddddZ< ddddZ=dS)r/a Dynamic graph Layer based on OOD, includes the parameters of the layer, the structure of the forward graph and so on. Parameters: name_scope (str, optional): prefix name used by the layer to name parameters. If prefix is "my_layer", parameter name in MyLayer can be "my_layer_0.w_n", where "w" is the parameter base name and "n" is an unique suffix auto-generated. If None, prefix name will be snake cased class name. Default: None. dtype(str, optional): data type of this parameter. If set str, it can be "bool", "float16", "float32", "float64", "int8", "int16", "int32", "int64", "uint8" or "uint16". Default: "float32" Returns: None Examples: .. code-block:: python >>> import paddle >>> paddle.seed(100) >>> class MyLayer(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear = paddle.nn.Linear(1, 1) ... self._dropout = paddle.nn.Dropout(p=0.5) ... ... def forward(self, input): ... temp = self._linear(input) ... temp = self._dropout(temp) ... return temp ... >>> x = paddle.randn([10, 1], 'float32') >>> mylayer = MyLayer() >>> mylayer.eval() # set mylayer._dropout to eval mode >>> out = mylayer(x) >>> mylayer.train() # set mylayer._dropout to train mode >>> out = mylayer(x) >>> print(out) Tensor(shape=[10, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[-3.44879317], [ 0. ], [ 0. ], [-0.73825276], [ 0. ], [ 0. ], [ 0.64444798], [-3.22185946], [ 0. ], [-0.68077987]]) rtrainingNfloat32 name_scope str | Noner`r*rrcCsd|_|durt|jj}t|}t||_t|j|_ d|_ ||_ t |_ t|_t|_t|_t|_t|_tggd|_i|_t|_t|_t|_d|_t|_t|_dS)NTF)r4rA)rrQrnrrJrgenerater8rg_helper_built_dtyperZ_init_in_dynamic_moder _parameters_buffersset!_non_persistable_buffer_names_set _sub_layers_loaddict_holderrr0r?_forward_pre_hooks_forward_post_hooks#_forward_pre_hooks_with_kwargs_flagZ_cast_to_low_precision_state_dict_hooksZ_original_funcs)rlrr`r;r;r<rjs4    zLayer.__init__rcC0tr td|_|D]}d|_q|S)a= Sets this Layer and all its sublayers to training mode. This only effects certain modules like `Dropout` and `BatchNorm`. Returns: Layer: self Examples: .. code-block:: python >>> import paddle >>> paddle.seed(100) >>> class MyLayer(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear = paddle.nn.Linear(1, 1) ... self._dropout = paddle.nn.Dropout(p=0.5) ... ... def forward(self, input): ... temp = self._linear(input) ... temp = self._dropout(temp) ... return temp ... >>> x = paddle.randn([10, 1], 'float32') >>> mylayer = MyLayer() >>> mylayer.eval() # set mylayer._dropout to eval mode >>> out = mylayer(x) >>> mylayer.train() # set mylayer._dropout to train mode >>> out = mylayer(x) >>> print(out) Tensor(shape=[10, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[-3.44879317], [ 0. ], [ 0. ], [-0.73825276], [ 0. ], [ 0. ], [ 0.64444798], [-3.22185946], [ 0. ], [-0.68077987]]) T)rr_dygraph_tracerZ train_moder sublayersrlr9r;r;r<trains 1  z Layer.traincCr)a Sets this Layer and all its sublayers to evaluation mode. This only effects certain modules like `Dropout` and `BatchNorm`. Returns: Layer: self Examples: .. code-block:: python >>> import paddle >>> paddle.seed(100) >>> class MyLayer(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear = paddle.nn.Linear(1, 1) ... self._dropout = paddle.nn.Dropout(p=0.5) ... ... def forward(self, input): ... temp = self._linear(input) ... temp = self._dropout(temp) ... return temp ... >>> x = paddle.randn([10, 1], 'float32') >>> mylayer = MyLayer() >>> mylayer.eval() # set mylayer._dropout to eval mode >>> out = mylayer(x) >>> print(out) Tensor(shape=[10, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[-1.72439659], [ 0.31532824], [ 0.01192369], [-0.36912638], [-1.63426113], [-0.93169814], [ 0.32222399], [-1.61092973], [ 0.77209264], [-0.34038994]]) F)rrrZ eval_moderrrr;r;r<evals -  z Layer.evalfnCallable[[Self], None]cCs$|D]}||q|||S)a Applies ``fn`` recursively to every sublayer (as returned by ``.sublayers()``) as well as self. Typical use includes initializing the parameters of a model. Parameters: fn (function): a function to be applied to each sublayer Returns: Layer, self Examples: .. code-block:: python >>> import paddle >>> import paddle.nn as nn >>> paddle.seed(2023) >>> net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2)) >>> def init_weights(layer): ... if type(layer) == nn.Linear: ... print('before init weight:', layer.weight.numpy()) ... new_weight = paddle.full(shape=layer.weight.shape, dtype=layer.weight.dtype, fill_value=0.9) ... layer.weight.set_value(new_weight) ... print('after init weight:', layer.weight.numpy()) ... >>> net.apply(init_weights) >>> print(net.state_dict()) before init weight: [[ 0.89611185 0.04935038] [-0.5888344 0.99266374]] after init weight: [[0.9 0.9] [0.9 0.9]] before init weight: [[-0.18615901 -0.22924072] [ 1.1517721 0.59859073]] after init weight: [[0.9 0.9] [0.9 0.9]] OrderedDict([('0.weight', Parameter containing: Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False, [[0.89999998, 0.89999998], [0.89999998, 0.89999998]])), ('0.bias', Parameter containing: Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False, [0., 0.])), ('1.weight', Parameter containing: Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False, [[0.89999998, 0.89999998], [0.89999998, 0.89999998]])), ('1.bias', Parameter containing: Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False, [0., 0.]))]) )childrenapply)rlrr9r;r;r<rEs 3 z Layer.applyrXcC|jS)a Full name for this layer, composed by name_scope + "/" + MyLayer.__class__.__name__ Returns: str, full name of this layer. Examples: .. code-block:: python >>> import paddle >>> class LinearNet(paddle.nn.Layer): ... def __init__(self): ... super().__init__(name_scope = "demo_linear_net") ... self._linear = paddle.nn.Linear(1, 1) ... ... def forward(self, x): ... return self._linear(x) ... >>> linear_net = LinearNet() >>> print(linear_net.full_name()) demo_linear_net_0 )r8rlr;r;r< full_nameszLayer.full_namehook_ForwardPostHookrcCt|j}||j|j<|S)aW Register a forward post-hook for Layer. The hook will be called after `forward` function has been computed. It should have the following form, `input` and `output` of the `hook` is `input` and `output` of the `Layer` respectively. User can use forward post-hook to change the output of the Layer or perform information statistics tasks on the Layer. hook(Layer, input, output) -> None or modified output Parameters: hook(function): a function registered as a forward post-hook Returns: HookRemoveHelper, a HookRemoveHelper object that can be used to remove the added hook by calling `hook_remove_helper.remove()` . Examples: .. code-block:: python >>> import paddle >>> import numpy as np >>> # the forward_post_hook change the output of the layer: output = output * 2 >>> def forward_post_hook(layer, input, output): ... # user can use layer, input and output for information statistics tasks ... ... # change the output ... return output * 2 ... >>> linear = paddle.nn.Linear(13, 5) >>> # register the hook >>> forward_post_hook_handle = linear.register_forward_post_hook(forward_post_hook) >>> value1 = np.arange(26).reshape(2, 13).astype("float32") >>> in1 = paddle.to_tensor(value1) >>> out0 = linear(in1) >>> # remove the hook >>> forward_post_hook_handle.remove() >>> out1 = linear(in1) >>> # hook change the linear's output to output * 2, so out0 is equal to out1 * 2. >>> assert (out0.numpy() == (out1.numpy()) * 2).any() )rrrrlrhook_remove_helperr;r;r<register_forward_post_hooks 2 z Layer.register_forward_post_hookF) with_kwargs_ForwardPreHookrcCs0t|j|jd}||j|j<|rd|j|j<|S)aO Register a forward pre-hook for Layer. The hook will be called before `forward` function has been computed. It should have the following form, `input` of the `hook` is `input` of the `Layer`, hook can either return a tuple or a single modified value in the hook. We will wrap the value into a tuple if a single value is returned(unless that value is already a tuple). User can use forward pre-hook to change the input of the Layer or perform information statistics tasks on the Layer. hook(Layer, input) -> None or modified input Parameters: hook(function): a function registered as a forward pre-hook Returns: HookRemoveHelper, a HookRemoveHelper object that can be used to remove the added hook by calling `hook_remove_helper.remove()` . Examples: .. code-block:: python >>> import paddle >>> import numpy as np >>> # the forward_pre_hook change the input of the layer: input = input * 2 >>> def forward_pre_hook(layer, input): ... # user can use layer and input for information statistics tasks ... ... # change the input ... input_return = (input[0] * 2) ... return input_return ... >>> linear = paddle.nn.Linear(13, 5) >>> # register the hook >>> forward_pre_hook_handle = linear.register_forward_pre_hook(forward_pre_hook) >>> value0 = np.arange(26).reshape(2, 13).astype("float32") >>> in0 = paddle.to_tensor(value0) >>> out0 = linear(in0) >>> # remove the hook >>> forward_pre_hook_handle.remove() >>> value1 = value0 * 2 >>> in1 = paddle.to_tensor(value1) >>> out1 = linear(in1) >>> # hook change the linear's input to input * 2, so out0 is equal to out1. >>> assert (out0.numpy() == out1.numpy()).any() rT)rrrr)rlrrrr;r;r<register_forward_pre_hooks5 zLayer.register_forward_pre_hookshaper-attrParamAttrLike | NoneDTypeLike | Noneis_biasdefault_initializerInitializer | NonedevicePlaceLike | Noner cCs8t|}t|tr|dkrd}|jj||||||dS)aB Create parameters for this layer. Parameters: shape(list): Shape of the parameter. The data type in the list must be int. attr(ParamAttr, optional): Parameter attribute of weight. Please refer to :ref:`api_paddle_ParamAttr`. Default: None. dtype(str, optional): Data type of this parameter. If set str, it can be "bool", "float16", "float32", "float64", "int8", "int16", "int32", "int64", "uint8" or "uint16". Default: "float32". is_bias(bool, optional): if this is a bias parameter. Default: False. default_initializer(Initializer, optional): the default initializer for this parameter. If set None, default initializer will be set to paddle.nn.initializer.Xavier and paddle.nn.initializer.Constant for non-bias and bias parameter, respectively. Default: None. device(PlaceLike, optional): the device place for the parameter. Default: None. Returns: :Tensor, created parameter. Examples: .. code-block:: python >>> import paddle >>> paddle.seed(2023) >>> class MyLayer(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear = paddle.nn.Linear(1, 1) ... w_tmp = self.create_parameter([1,1]) ... self.add_parameter("w_tmp", w_tmp) ... ... def forward(self, input): ... return self._linear(input) ... >>> mylayer = MyLayer() >>> for name, param in mylayer.named_parameters(): ... print(name, param) # will print w_tmp,_linear.weight,_linear.bias w_tmp Parameter containing: Tensor(shape=[1, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[0.06979191]]) _linear.weight Parameter containing: Tensor(shape=[1, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[1.26729357]]) _linear.bias Parameter containing: Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=False, [0.]) N)r)r~rrtrXrcreate_parameter)rlrrr`rrrZ temp_attrr;r;r<rs 7 zLayer.create_parameterz2.0.0zpaddle.nn.Layer.create_tensorz(New api in create_tensor, easier to use.)ZsinceZ update_toreasonrO persistable bool | NonecCP|dur d|j|g}n td|jdg}|jjj|||tj j j dS)a, Create Tensor for this layer. Parameters: name(str, optional): name of the tensor. Please refer to :ref:`api_guide_Name` . Default: None persistable(bool, optional): if set this tensor persistable. Default: False dtype(str, optional): data type of this parameter. If set str, it can be "bool", "float16", "float32", "float64","int8", "int16", "int32", "int64", "uint8" or "uint16". If set None, it will be "float32". Default: None Returns: Tensor, created Tensor. Examples: .. code-block:: python >>> import paddle >>> class MyLinear(paddle.nn.Layer): ... def __init__(self, ... in_features, ... out_features): ... super().__init__() ... self.linear = paddle.nn.Linear( 10, 10) ... ... self.back_var = self.create_variable(name = "linear_tmp_0", dtype=self._dtype) ... ... def forward(self, input): ... out = self.linear(input) ... paddle.assign( out, self.back_var) ... ... return out N._generated_varrOrr`rc rYr8rrrrqr3Z create_varr rVarTypeZ DENSE_TENSORrlrOrr`var_namer;r;r<create_variableOs. zLayer.create_variablecCr)a] Create Tensor for this layer. Parameters: name(str, optional): name of the tensor. Please refer to :ref:`api_guide_Name` . Default: None. persistable(bool, optional): if set this tensor persistable. Default: False. dtype(str, optional): data type of this parameter. If set str, it can be "bool", "float16", "float32", "float64", "int8", "int16", "int32", "int64", "uint8" or "uint16". If set None, it will be "float32". Default: None. Returns: Tensor, created Tensor. Examples: .. code-block:: python >>> import paddle >>> class MyLinear(paddle.nn.Layer): ... def __init__(self, ... in_features, ... out_features): ... super().__init__() ... self.linear = paddle.nn.Linear(10, 10) ... ... self.back_var = self.create_tensor(name = "linear_tmp_0", dtype=self._dtype) ... ... def forward(self, input): ... out = self.linear(input) ... paddle.assign(out, self.back_var) ... ... return out Nrrrrrr;r;r< create_tensors* zLayer.create_tensorTrb list[Tensor]cCdd|j|dD}|S)a Returns a list of all Parameters from current layer and its sub-layers. Parameters: include_sublayers (bool, optional): Whether to return the parameters of the sublayer. If True, the returned list contains the parameters of the sublayer. Default: True. Returns: list, list of Tensor, a list of Parameters. Examples: .. code-block:: python >>> import paddle >>> paddle.seed(100) >>> linear = paddle.nn.Linear(1, 1) >>> print(linear.parameters()) [Parameter containing: Tensor(shape=[1, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[0.18551230]]), Parameter containing: Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=False, [0.])] cSg|]\}}|qSr;r;).0_rr;r;r< z$Layer.parameters..rb)named_parametersrlrbrxr;r;r< parameterss zLayer.parameterscCsgd}t|tjtjfst|turU||vrUt|ttjfr$t|}||_| D]}||_q+|j ddD] \}}| d|q7|j ddD] \}}| d|qH|Stdt|)a Casts all parameters and buffers to dtype and then return the Layer. Parameters: dtype(str|paddle.dtype|numpy.dtype): target data type of layer. If set str, it can be "bool", "bfloat16", "float16", "float32", "float64", "int8", "int16", "int32", "int64", "uint8", "complex64", "complex128". Default: None Returns: Layer, self Examples: .. code-block:: python >>> import paddle >>> import paddle.nn as nn >>> weight_attr = paddle.ParamAttr(name="weight",initializer=paddle.nn.initializer.Constant(value=1.5)) >>> bias_attr = paddle.ParamAttr(name="bias",initializer=paddle.nn.initializer.Constant(value=2.5)) >>> linear = paddle.nn.Linear(2, 2, weight_attr=weight_attr, bias_attr=bias_attr).to(device="cpu",dtype="float32") >>> print(linear) Linear(in_features=2, out_features=2, dtype=float32) >>> print(linear.parameters()) [Parameter containing: Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=False, [[1.50000000, 1.50000000], [1.50000000, 1.50000000]]), Parameter containing: Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=False, [2.50000000, 2.50000000])] >>> linear=linear.astype("int8") >>> print(linear) Linear(in_features=2, out_features=2, dtype=paddle.int8) >>> print(linear.parameters()) [Parameter containing: Tensor(shape=[2, 2], dtype=int8, place=Place(cpu), stop_gradient=False, [[1, 1], [1, 1]]), Parameter containing: Tensor(shape=[2], dtype=int8, place=Place(cpu), stop_gradient=False, [2, 2])] ) bfloat16float16rZfloat64Zint8Zint16Zint32Zint64Zuint8Z complex64Z complex128rTrNzdtype value error, must be 'bfloat16', 'float16', 'float32', 'float64', 'int8', 'int16', 'int32', 'int64', 'uint8', 'complex64', 'complex128', 'bool', or paddle.dtype, numpy.dtype, but receive )rtpaddler`nprcrXrrrrrZ_to named_bufferstor|)rlr`Z valid_dtypesr9rrbufferr;r;r<astypes(-   z Layer.astypeIterable[Layer]ccs|D]\}}|VqdS)ar Returns an iterator over immediate children layers. Yields: Layer: a child layer Examples: .. code-block:: python >>> import paddle >>> linear1 = paddle.nn.Linear(10, 3) >>> linear2 = paddle.nn.Linear(3, 10, bias_attr=False) >>> model = paddle.nn.Sequential(linear1, linear2) >>> layer_list = list(model.children()) >>> print(layer_list) [Linear(in_features=10, out_features=3, dtype=float32), Linear(in_features=3, out_features=10, dtype=float32)] N)named_children)rlrr9r;r;r<r8szLayer.childrenIterable[tuple[str, Layer]]ccsDt}|jD]\}}|dur||vr||||fVq dS)aReturns an iterator over immediate children layers, yielding both the name of the layer as well as the layer itself. Yields: (string, Layer): Tuple containing a name and child layer Examples: .. code-block:: python >>> import paddle >>> linear1 = paddle.nn.Linear(10, 3) >>> linear2 = paddle.nn.Linear(3, 10, bias_attr=False) >>> model = paddle.nn.Sequential(linear1, linear2) >>> for prefix, layer in model.named_children(): ... print(prefix, layer) 0 Linear(in_features=10, out_features=3, dtype=float32) 1 Linear(in_features=3, out_features=10, dtype=float32) N)rrr@add)rlmemorOr9r;r;r<rRs  zLayer.named_children include_self list[Layer]cCr)a Returns a list of sub layers. Parameters: include_self(bool, optional): Whether return self as sublayers. Default: False. Returns: list of Layer, a list of sub layers. Examples: .. code-block:: python >>> import paddle >>> class MyLayer(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear = paddle.nn.Linear(1, 1) ... self._dropout = paddle.nn.Dropout(p=0.5) ... ... def forward(self, input): ... temp = self._linear(input) ... temp = self._dropout(temp) ... return temp ... >>> mylayer = MyLayer() >>> print(mylayer.sublayers()) [Linear(in_features=1, out_features=1, dtype=float32), Dropout(p=0.5, axis=None, mode=upscale_in_train)] cSrr;r;)rrr9r;r;r<rrz#Layer.sublayers..)r)named_sublayers)rlrrxr;r;r<rls zLayer.sublayersrprefixremove_duplicateIterable[tuple[str, Tensor]]c cstr ts tnt}|r|j|d|dnt|g|g}|D]/\}}|j}|D]#\} } | dus7| |vr8q+|r?|| ||rDdnd| } | | fVq+q dS)a Returns an iterator over all parameters in the Layer, yielding tuple of name and parameter. Parameters: prefix(str, optional): Prefix to prepend to all parameter names. Default: ''. include_sublayers(bool, optional): Whether include the parameters of sublayers. If True, also include the named parameters from sublayers. Default: True. remove_duplicate(bool, optional): Whether to remove duplicated parameters in the result. Default: True. Yields: (string, Parameter): Tuple of name and Parameter Examples: .. code-block:: python >>> import paddle >>> paddle.seed(100) >>> fc1 = paddle.nn.Linear(10, 3) >>> fc2 = paddle.nn.Linear(3, 10, bias_attr=False) >>> model = paddle.nn.Sequential(fc1, fc2) >>> for name, param in model.named_parameters(): ... print(name, param) 0.weight Parameter containing: Tensor(shape=[10, 3], dtype=float32, place=Place(cpu), stop_gradient=False, [[ 0.07276392, -0.39791510, -0.66356444], [ 0.02143478, -0.18519843, -0.32485050], [-0.42249614, 0.08450919, -0.66838276], [ 0.38208580, -0.24303678, 0.55127048], [ 0.47745085, 0.62117910, -0.08336520], [-0.28653207, 0.47237599, -0.05868882], [-0.14385653, 0.29945642, 0.12832761], [-0.21237159, 0.38539791, -0.62760031], [ 0.02637231, 0.20621127, 0.43255770], [-0.19984481, -0.26259184, -0.29696006]]) 0.bias Parameter containing: Tensor(shape=[3], dtype=float32, place=Place(cpu), stop_gradient=False, [0., 0., 0.]) 1.weight Parameter containing: Tensor(shape=[3, 10], dtype=float32, place=Place(cpu), stop_gradient=False, [[ 0.01985580, -0.40268910, 0.41172385, -0.47249708, -0.09002256, -0.00533628, -0.52048630, 0.62360322, 0.20848787, -0.02033746], [ 0.58281910, 0.12841827, 0.12907702, 0.02325618, -0.07746267, 0.31950659, -0.37924835, -0.59209681, -0.11732036, -0.58378261], [-0.62100595, 0.22293305, 0.28229684, -0.03687060, -0.59323978, 0.08411229, 0.53275704, 0.40431368, 0.03171402, -0.17922515]]) TrrrNrr) rrr rrrrr@r) rlrrbrZ params_setr layer_prefixsublayerparamskeyrrOr;r;r<rs,7      zLayer.named_parameters layers_setset[Layer] | Noneccs|durt}|r||vr|r||||fV|jD]\}}|dur(q||r-dnd|}|j|d||dEdHqdS)a Returns an iterator over all sublayers in the Layer, yielding tuple of name and sublayer. The duplicate sublayer will only be yielded once. Parameters: prefix(str, optional): Prefix to prepend to all parameter names. Default: ''. include_self(bool, optional): Whether include the Layer itself. Default: False. layers_set(set, optional): The set to record duplicate sublayers. Default: None. remove_duplicate(bool, optional): Whether to remove duplicated sublayers in the result. Default: True. Yields: (string, Layer): Tuple of name and Layer Examples: .. code-block:: python >>> import paddle >>> fc1 = paddle.nn.Linear(10, 3) >>> fc2 = paddle.nn.Linear(3, 10, bias_attr=False) >>> model = paddle.nn.Sequential(fc1, fc2) >>> for prefix, layer in model.named_sublayers(): ... print(prefix, layer) 0 Linear(in_features=10, out_features=3, dtype=float32) 1 Linear(in_features=3, out_features=10, dtype=float32) >>> l = paddle.nn.Linear(10, 3) >>> model = paddle.nn.Sequential(l, l) >>> for prefix, layer in model.named_sublayers(include_self=True, remove_duplicate=True): ... print(prefix, layer) Sequential( (0): Linear(in_features=10, out_features=3, dtype=float32) (1): Linear(in_features=10, out_features=3, dtype=float32) ) 0 Linear(in_features=10, out_features=3, dtype=float32) >>> l = paddle.nn.Linear(10, 3) >>> model = paddle.nn.Sequential(l, l) >>> for prefix, layer in model.named_sublayers(include_self=True, remove_duplicate=False): ... print(prefix, layer) Sequential( (0): Linear(in_features=10, out_features=3, dtype=float32) (1): Linear(in_features=10, out_features=3, dtype=float32) ) 0 Linear(in_features=10, out_features=3, dtype=float32) 1 Linear(in_features=10, out_features=3, dtype=float32) NrrT)rrrr)rrrr@r)rlrrrrr r9r r;r;r<rs$8   zLayer.named_sublayerstensorcCsd|jvr tdt|tstdt|jdd|vr!td|dkr)tdt||r;||j vr;td|d |d urRt|t j j ksRtd t|jd||j |<|ra|j |d S|j |d S) a Registers a tensor as buffer into the layer. `buffer` is a non-trainable tensor and will not be updated by optimizer, but is necessary for evaluation and inference. For example, the mean and variance in BatchNorm layers. The registered buffer is persistable by default, and will be saved into `state_dict` alongside parameters. If set persistable=False, it registers a non-persistable buffer, so that it will not be a part of `state_dict` . Buffers can be accessed as attributes using given names. Parameters: name (string): name of the buffer. The buffer can be accessed from this layer using the given name tensor (Tensor): the tensor to be registered as buffer. persistable (bool): whether the buffer is part of this layer's state_dict. Returns: None Examples: .. code-block:: python >>> import numpy as np >>> import paddle >>> linear = paddle.nn.Linear(10, 3) >>> value = np.array([0]).astype("float32") >>> buffer = paddle.to_tensor(value) >>> linear.register_buffer("buf_name", buffer, persistable=True) >>> # get the buffer by attribute. >>> print(linear.buf_name) Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True, [0.]) r)super().__init__() should be called firstz4The name of buffer should be a string, but received rzThe name of buffer can not contain `.`, because when you access the newly added buffer in the form of `self.**.**`, it will cause AttributeError.rz$The name of buffer can not be empty.z attribute '' already exists.Nz>The registered buffer should be a Paddle.Tensor, but received )__dict__r|rtrXrrcrKeyErrorhasattrrr eagerr rdiscardr)rlrOrrr;r;r<register_buffer's, *  zLayer.register_buffercCr)a\ Returns a list of all buffers from current layer and its sub-layers. Parameters: include_sublayers(bool, optional): Whether include the buffers of sublayers. If True, also include the buffers from sublayers. Default: True. Returns: list of Tensor, a list of buffers. Examples: .. code-block:: python >>> import numpy as np >>> import paddle >>> linear = paddle.nn.Linear(10, 3) >>> value = np.array([0]).astype("float32") >>> buffer = paddle.to_tensor(value) >>> linear.register_buffer("buf_name", buffer, persistable=True) >>> print(linear.buffers()) [Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True, [0.])] cSrr;r;)rrrr;r;r<rrz!Layer.buffers..r)rrr;r;r<buffersls z Layer.buffersc cst}|r|j|d|dnt|g|g}|D]/\}}|j}|D]#\} } | dus.| |vr/q"|r6|| ||r;dnd| } | | fVq"qdS)a Returns an iterator over all buffers in the Layer, yielding tuple of name and Tensor. Parameters: prefix(str, optional): Prefix to prepend to all buffer names. Default: ''. include_sublayers(bool, optional): Whether include the buffers of sublayers. If True, also include the named buffers from sublayers. Default: True. remove_duplicate(bool, optional): Whether to remove duplicated buffers in the result. Default: True. Yields: (string, Tensor): Tuple of name and tensor Examples: .. code-block:: python >>> import numpy as np >>> import paddle >>> fc1 = paddle.nn.Linear(10, 3) >>> buffer1 = paddle.to_tensor(np.array([0]).astype("float32")) >>> # register a tensor as buffer by specific `persistable` >>> fc1.register_buffer("buf_name_1", buffer1, persistable=True) >>> fc2 = paddle.nn.Linear(3, 10) >>> buffer2 = paddle.to_tensor(np.array([1]).astype("float32")) >>> # register a buffer by assigning an attribute with Tensor. >>> # The `persistable` can only be False by this way. >>> fc2.buf_name_2 = buffer2 >>> model = paddle.nn.Sequential(fc1, fc2) >>> # get all named buffers >>> for name, buffer in model.named_buffers(): ... print(name, buffer) 0.buf_name_1 Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True, [0.]) 1.buf_name_2 Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=True, [1.]) Tr Nrr)rrrrr@r) rlrrbrZ buffers_setrr r rr rrOr;r;r<rs*.     zLayer.named_buffers set_to_zerocCs"|D] }|jr||qdS)ad Clear the gradients of all parameters for this layer. Args: set_to_zero (bool, optional): Whether to set the trainable parameters' gradients to zero or None. Default is True. Returns: None Examples: .. code-block:: python >>> import paddle >>> import numpy as np >>> value = np.arange(26).reshape(2, 13).astype("float32") >>> a = paddle.to_tensor(value) >>> linear = paddle.nn.Linear(13, 5) >>> adam = paddle.optimizer.Adam(learning_rate=0.01, ... parameters=linear.parameters()) >>> out = linear(a) >>> out.backward() >>> adam.step() >>> linear.clear_gradients() N)rZ trainableZclear_gradient)rlrpr;r;r<clear_gradientss  zLayer.clear_gradientsargsrkwargscOsdSrsr;)rlrrr;r;r< _build_onceszLayer._build_oncer:c OsZ|jD]?\}}||jvr1||||}|dur0t|tr(t|dkr(|\}}qtd|dq|||}|durDt|tsB|f}|}q|jsS|j|i|d|_t ryt |j j t jj|j|i|}Wdn1sswYnt|j j |j|i|}Wdn1swY|jD]}||||}|dur|}q|S)NzPforward pre-hook must return None or a tuple of (new_args, new_kwargs), but got rT)rr@rrtrvr2 RuntimeErrorrrr&r Z RecordEventrnrZTracerEventTypeForwardforwardr rvalues) rlr:rZhook_idZforward_pre_hookZargs_kwargs_result hook_resultrCZforward_post_hookr;r;r<_dygraph_call_funcsN        zLayer._dygraph_call_funccOsXts$|js$|js$|jjtjus|jr$tr$trt r$|j |i|S|j |i|Srs) rrrrnrr/rrr&rr#r&rlr:rr;r;r<__call__szLayer.__call__cOst)z Defines the computation performed at every call. Should be overridden by all subclasses. Parameters: *inputs(tuple): unpacked tuple arguments **kwargs(dict): unpacked dict arguments )NotImplementedErrorr'r;r;r<r#,s z Layer.forwardcGstd)Nz"Layer shouldn't implement backward)r|)rlr:r;r;r<backward7szLayer.backwardr cCs$t|ts |dus J||j|<|S)a+ Adds a sub Layer instance. Added sublayer can be accessed by self.name Parameters: name(str): name of this sublayer. sublayer(Layer): an instance of Layer. Returns: Layer, the sublayer passed in. Examples: .. code-block:: python >>> import paddle >>> class MySequential(paddle.nn.Layer): ... def __init__(self, *layers): ... super().__init__() ... if len(layers) > 0 and isinstance(layers[0], tuple): ... for name, layer in layers: ... self.add_sublayer(name, layer) ... else: ... for idx, layer in enumerate(layers): ... self.add_sublayer(str(idx), layer) ... ... def forward(self, input): ... for layer in self._sub_layers.values(): ... input = layer(input) ... return input ... >>> fc1 = paddle.nn.Linear(10, 3) >>> fc2 = paddle.nn.Linear(3, 10, bias_attr=False) >>> model = MySequential(fc1, fc2) >>> for prefix, layer in model.named_sublayers(): ... print(prefix, layer) 0 Linear(in_features=10, out_features=3, dtype=float32) 1 Linear(in_features=3, out_features=10, dtype=float32) N)rtr/r)rlrOr r;r;r< add_sublayer:s) zLayer.add_sublayer parametercCsd|jvr tdt|tstdt|jdd|vr!td|dkr)tdt||r;||j vr;td|d |d urTt|t j t j jfsTtd t|jd|d ur]d |j |<t|jd kr||j|jvssJd |jd||j|j||j |<|S)aAdds a Parameter instance. Added parameter can be accessed by self.name Parameters: name(str): name of this sublayer. parameter(Parameter): an instance of Parameter. Returns: Parameter, the parameter passed in. Examples: .. code-block:: python >>> import paddle >>> paddle.seed(100) >>> class MyLayer(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self._linear = paddle.nn.Linear(1, 1) ... w_tmp = self.create_parameter([1,1]) ... self.add_parameter("w_tmp", w_tmp) ... ... def forward(self, input): ... return self._linear(input) ... >>> mylayer = MyLayer() >>> for name, param in mylayer.named_parameters(): ... print(name, param) w_tmp Parameter containing: Tensor(shape=[1, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[-1.01448846]]) _linear.weight Parameter containing: Tensor(shape=[1, 1], dtype=float32, place=Place(cpu), stop_gradient=False, [[0.18551230]]) _linear.bias Parameter containing: Tensor(shape=[1], dtype=float32, place=Place(cpu), stop_gradient=False, [0.]) rz,super().__init__() should be called firstly.z7The name of parameter should be a string, but received rzThe name of parameter can not contain `.`, because when you access the newly added parameter in the form of `self.**.**`, it will cause AttributeError.rz'The name of parameter can not be empty.zThe parameter 'rNz>The parameter to be added should be a Parameter, but received r&Parameter not found, Can't not find [  ] in state_dict)rr!rtrXrrcrrrrrrrpirValuer2rrO set_value)rlrOr,r;r;r< add_parameterhs: '     zLayer.add_parametercsfdd}t|tstdt|jj||dds4t}t j j dks/J|gj _ |ddsa t }t jdkrNjj|jdd t j j dksXJj j |d Sd S) aY Add customized attribute while append_op. In case of quantization, we want to save some attributes into op_desc while exporting inference model by @to_static. Arguments: attrs(dict): customized attributes that will be added into op_descs. NOTE: The interface is only exposed to developers. cs@|rjnj}|r tnt}d}|rtt|}|||k}|S)NF)rrr=rGnextreversed) is_pre_hookZ layers_hooksZcandidate_hookZalready_registeredlast_keyrr;r<is_already_registereds  z2Layer._set_op_attrs..is_already_registeredz)attrs should be type(dict), but received T)r5rFrS)lastN)rtdictrrcrr?updaterr=r2r0rArrGr move_to_endrrW)rlror7Zpre_hook_helperZpost_hook_helperr;rr< _set_op_attrss.     zLayer._set_op_attrsdict[str, Any]cCrrs)rrr;r;r< __getstate__szLayer.__getstate__statecCs|j|dSrs)rr:)rlr?r;r;r< __setstate__szLayer.__setstate__cCsd|jvr|jd}||jvrtrt|j|S|j|Sd|jvr2|jd}||jvr2|j|Sd|jvrM|jd}||vrMtrIt||S||St||S)Nrrr)rrrrrobject__getattribute__)rlrOrrrr;r;r< __getattr__s"            zLayer.__getattr__valuecspfdd}t|tjjjjrt|||_dStt t |dt r-t|||j dd}t|tjrp|durBtdt|jdkra|j|jvsXJd|jd||j|j||j |j|j||<dSt|tjjr|dkr|durtd||j |j|j||<dS|dur|vr|durtd d t |jd d|<dS|j d d}t|tr|durtd||j |j|j||<dS|dur|vr|durtd dt |jd d|<dS|j dd}t|tjj r8|durtd||j |j|j|jvr&|j!"|js2t#$d|_||<dS|dur|vrt |tj%ksRt|tjjrddlm&}t'rq|durqt(dddd|dust t |tjj kr|||<dS||t |dS|durtddt |jd d|<dSt||dS)Ncs|D] }|vr |=qdSrsr;)ZdictsdrOr;r<_remove_if_exist s z+Layer.__setattr__.._remove_if_existrrrr-r.zbuiltin.parameterzassignment to parameter 'z0' should be of type Parameter or None, but got ''rzassignment to sublayer 'z,' should be of type Layer or None, but got 'rZ _buffers_)assignzIn Dy2stat, self.z is a buffer and self.z0 is not allowed to be set to Variable when self.z is None.zassignment to buffers 'z7' should be of type core.DenseTensor or None, but got '))rtrZjitZ dy2staticZprogram_translatorZStaticFunctionrA __setattr__Z _patched_namegetattrrcpropertyrrHrrr|r2rrOr1rrr/r0Zget_defining_oprrr/rr rr rrrrVariablerIrr!)rlrOrDrGr ZlayersrrIr;rFr<rJs                   zLayer.__setattr__cCs^||jvr |j|=dS||jvr|j|=dS||jvr'|j|=|j|dSt||dSrs)rrrrrrA __delattr__rkr;r;r<rNvs     zLayer.__delattr__ list[str]cCsZt|j}t|j}t|j}t|j}t|j}|||||}|S)a Return a list. Get all parameters, buffers(non-parameter tensors), sublayers, method and attr of Layer. Examples: .. code-block:: python >>> import paddle >>> import numpy as np >>> class Mylayer(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self.linear1 = paddle.nn.Linear(10, 10) ... self.linear2 = paddle.nn.Linear(5, 5) ... self.conv2d = paddle.nn.Conv2D(3, 2, 3) ... self.embedding = paddle.nn.Embedding(128, 16) ... self.h_0 = paddle.to_tensor(np.zeros([10, 10]).astype('float32')) ... >>> mylayer = Mylayer() >>> print(dir(mylayer)) ['__call__', '__class__', '__delattr__', '__dict__', ..., 'training'] )dirrnrurkeysrrr)rlmethodrorrrrQr;r;r<__dir__s z Layer.__dir__cCsdS)zs Extra representation of this layer, you can have custom implementation of your own layer. rr;rr;r;r< extra_reprszLayer.extra_reprcCsg}|}|d}g}|jD]\}}t|}t|d}|d|d|q|jjd}|rQt |dkrE|dd |d7}n t |dkrQ||d7}|r^|dd |d7}|d7}|S) NrRr (z): rSz r)) rTrUrr@reprr_rWrnrr2rY)rl extra_linesrTZsublayer_linesrOr9Z sublayer_strZ final_strr;r;r<__repr__s$      zLayer.__repr___StateDictHookcCrrs)rrrrr;r;r<register_state_dict_hooks  zLayer.register_state_dict_hook destination_StateDict | Nonestructured_name_prefix _StateDictc Cs|durt}|jD]\}}|dur||||<q |jD]\}}|dur3||jvr3||||<q |rO|jD]\}}|durN|||||dq;|S)z The difference from state_dict() is that state_dict_hook will not be called, but the original types of parameters and buffers will be maintained. Nr)rrr@rrr_obtain_parameters_buffers) rlr\rbr^rOdatar layer_name layer_itemr;r;r<r`s*     z Layer._obtain_parameters_buffersinclude_non_persistable_bufferuse_hook keep_varsc Cs|durt}|jD]\}}|dur |r|n||||<q |jD],\}} |sB| durA||jvrA|r9| n| |||<q&| durR|rJ| n| |||<q&|rq|jD]\} } | durp| |||| d|||qZ|r|j D] } | |} | dur| }qx|S)a| Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict Parameters: destination(dict, optional) : If provide, all the parameters and persistable buffers will be set to this dict . Default: None. include_sublayers(bool, optional) : If true, also include the parameters and persistable buffers from sublayers. Default: True. include_non_persistable_buffer(bool, optional): If true, include non persistable buffers of current layer and its sub-layers, it is used in pure fp16 and jit.save. Default: False. use_hook(bool, optional) : If true, the operations contained in _state_dict_hooks will be appended to the destination. Default: True. keep_vars(bool, optional) : If false, the returned tensors in the state dict are detached from autograd. Default: True. Nr) rrr@detachrrr_state_dict_implrr$)rlr\rbr^rdrerfrOrarrbrcZstate_dict_hookr%r;r;r<rhsJ      zLayer._state_dict_implcC|j|||d||dS)a" Get all parameters and buffers of current layer and its sub-layers. And set them into a dict Parameters: destination(dict, optional) : If provide, all the parameters and persistable buffers will be set to this dict . Default: None. include_sublayers(bool, optional) : If true, also include the parameters and persistable buffers from sublayers. Default: True. use_hook(bool, optional) : If true, the operations contained in _state_dict_hooks will be appended to the destination. Default: True. keep_vars(bool, optional) : If false, the returned tensors in the state dict are detached from autograd. Default: True. Returns: dict, a dict contains all the parameters and persistable buffers. Examples: .. code-block:: python >>> import paddle >>> emb = paddle.nn.Embedding(10, 10) >>> state_dict = emb.to_static_state_dict() >>> paddle.save( state_dict, "paddle_dy.pdparams") Tr\rbr^rdrerfrhrlr\rbr^rerfr;r;r<to_static_state_dict$s zLayer.to_static_state_dictcCri)a" Get all parameters and persistable buffers of current layer and its sub-layers. And set them into a dict Parameters: destination(dict, optional) : If provide, all the parameters and persistable buffers will be set to this dict . Default: None. include_sublayers(bool, optional) : If true, also include the parameters and persistable buffers from sublayers. Default: True. use_hook(bool, optional) : If true, the operations contained in _state_dict_hooks will be appended to the destination. Default: True. keep_vars(bool, optional) : If false, the returned tensors in the state dict are detached from autograd. Default: True. Returns: dict: a dict contains all the parameters and persistable buffers. Examples: .. code-block:: python >>> import paddle >>> emb = paddle.nn.Embedding(10, 10) >>> state_dict = emb.state_dict() >>> paddle.save(state_dict, "paddle_dy.pdparams") Frjrkrlr;r;r< state_dictMszLayer.state_dictr#cCsji}|jddd}t|d|d}|||jD]\}}|dur2|j||dd}||q|S)aRecursively builds a sharded state dictionary for the model and its sub-layers. Args: structured_name_prefix: Prefix to prepend to all tensor names for hierarchical naming. Returns: Dictionary mapping tensor names to ShardedWeight. The dictionary contains both the current layer's parameters and all sub-layer parameters. rF)r^rbN)rnZ shard_rulesrr)r^)rnr$r:rr@sharded_state_dict)rlr^rornZcurrent_sharded_dictrbrcZ sub_shardedr;r;r<rous&    zLayer.sharded_state_dictrnuse_structured_nametuple[list[str], list[str]]c sgtg}fdd}g}|jddD]6\}}|r!|n|j}z |||} || WqtyO} ztd|dt| WYd} ~ qd} ~ ww D] }|vr_||qTt rs|D] \}} | | qe|fSdd } zAt rt tjjj} tjjjd d |Dt| nt tj} td d |Dt| |D] \}} | || qW|fSty}ztd d}~wty}ztd d}~ww)a Set parameters and persistable buffers from state_dict. All the parameters and buffers will be reset by the tensor in the state_dict Parameters: state_dict(dict) : Dict contains all the parameters and persistable buffers. use_structured_name(bool, optional) : If true, use structured name as key, otherwise, use parameter or buffer name as key. Default: True. Returns: missing_keys(list):A list of str containing the missing keys unexpected_keys(list):A list of str containing the unexpected keys Examples: .. code-block:: python >>> import paddle >>> emb = paddle.nn.Embedding(10, 10) >>> state_dict = emb.state_dict() >>> paddle.save(state_dict, "paddle_dy.pdparams") >>> para_state_dict = paddle.load("paddle_dy.pdparams") >>> emb.set_state_dict(para_state_dict) cs|d}|dur|t|dt|ttfrCt|t|kr:|t|dt|dt||||fSt |j rM| n|j }t|t|j krp|t|dt|dt|j d|||fS)Nz# is not found in the provided dict.z receives the length of z, but the expected shape is z receives a shape r) rHrWr|rtr9rur2rinspectismethodr)r rr?Z state_shapeZ match_keysZ missing_keysrnr;r< _check_matchs2        z*Layer.set_state_dict.._check_matchF)rezSkip loading for z. NcSst|j}|}|rt}nR|rt }nI| r6t }| |t |}n2|rVt }| |ttjdd|}nt }| |t|}|||dS)N:r)rZfind_varrO get_tensorZ_placeZ is_cpu_placer CPUPlaceZis_cuda_pinned_placeCUDAPinnedPlaceZ is_xpu_placeZPlaceZ set_placeXPUPlaceZ xpu_device_idZis_custom_placeZ CustomPlacerrZ get_devicerUZcustom_device_id CUDAPlaceZ gpu_device_idr)rZndarraytrplacer;r;r<_set_vars*  z&Layer.set_state_dict.._set_varcSg|]\}}|qSr;r;rrr?r;r;r<r z(Layer.set_state_dict..cSrr;r;rr;r;r<r rzThis error might happens in dy2static, while calling 'set_state_dict' dynamically in 'forward', which is not supported. If you only need call 'set_state_dict' once, move it to '__init__'.)rrhr@rOrWr|r6r7rXrQrr1rrrbaserZ_current_expected_place__default_executorZ libpaddler/Zcreate_loaded_parameterr _get_devicer Z_create_loaded_parameterr)rlrnrpZunexpected_keysruZmatched_param_stater rZkey_nameZ match_reserrr?r~executorer;rtr<set_state_dictsp &    8        zLayer.set_state_dictblockingcCs|j|||dddS)a Cast the parameters and buffers of Layer by the give device, dtype and blocking. Parameters: device(str|paddle.CPUPlace()|paddle.CUDAPlace()|paddle.CUDAPinnedPlace()|paddle.XPUPlace()|None, optional): The device of the Layer which want to be stored. If None, the device is the same with the original Tensor. If device is string, it can be ``cpu``, ``gpu:x`` and ``xpu:x``, where ``x`` is the index of the GPUs or XPUs. Default: None. dtype(str|numpy.dtype|paddle.dtype|None, optional): The type of the data. If None, the dtype is the same with the original Tensor. Default: None. blocking(bool|None, optional): If False and the source is in pinned memory, the copy will be asynchronous with respect to the host. Otherwise, the argument has no effect. If None, the blocking is set True. Default: None. Returns: self Examples: .. code-block:: python >>> import paddle >>> paddle.seed(2023) >>> linear=paddle.nn.Linear(2, 2) >>> linear.weight >>> print(linear.weight) Parameter containing: Tensor(shape=[2, 2], dtype=float32, place=Place(gpu:0), stop_gradient=False, [[ 0.89611185, 0.04935038], [-0.58883440, 0.99266374]]) >>> linear.to(dtype='float64') >>> linear.weight >>> print(linear.weight) Parameter containing: Tensor(shape=[2, 2], dtype=float64, place=Place(gpu:0), stop_gradient=False, [[ 0.89611185, 0.04935038], [-0.58883440, 0.99266374]]) >>> linear.to(device='cpu') >>> linear.weight >>> print(linear.weight) Parameter containing: Tensor(shape=[2, 2], dtype=float64, place=Place(cpu), stop_gradient=False, [[ 0.89611185, 0.04935038], [-0.58883440, 0.99266374]]) >>> # doctest: +REQUIRES(env:GPU) >>> linear.to(device=paddle.CUDAPinnedPlace(), blocking=False) >>> linear.weight >>> print(linear.weight) Parameter containing: Tensor(shape=[2, 2], dtype=float64, place=Place(gpu_pinned), stop_gradient=False, [[ 0.89611185, 0.04935038], [-0.58883440, 0.99266374]]) TF)rr`rrbra)rd)rlrr`rr;r;r<r" s>zLayer.tofuncICallable[[Tensor, PlaceLike | None, DTypeLike | None, bool | None], None]c Cs|r|D] }||||||q|jD]C\}}|durZt|||||} Wdn1s4wY|jdurZt|||||} Wdn1sUwYq|jD]\}} | durr|| ||||j|<q`||_dSrs) r_applyrr@rZgradZ _grad_ivarrr) rlrrr`rrbr9r rZ param_appliedZ grad_appliedbufr;r;r<rh s(    z Layer._applyr|cCsv|dur|j}|dur|j}t|tjtjfst|}|jr]t|tjr*t |n|}t |}t |j |dddd}t}||krZ|t|} |n|} n|} |dur|| jkrtjjj| jd| j|d} Wdn1swYn| } |dur| j|s| ||} n| } |} | } |r| | |S| | |S)NrSg333333?)r}r)r}r`rtrrr ZDataTyperZ is_gpu_placer!Z size_of_dtyperprodrgpu_memory_availableZ_copy_torrxrDrw_clearrrZ_dygraph_place_guardcastZ_equalsZ_is_initializedZ_share_data_withZ_share_data_nocheck_with)rlr|rr`rZ proto_dtypeZ size_dtypeZwaiting_alloc_memoryrZt_usedZt_castedZnew_tZ dst_tensorZ src_tensorr;r;r< _transform sL        zLayer._transformracs|dur|dur|durS|dur5t|trtj|}nt|tjtjtjtj fr,n t dt |j |dur.transformignore)category)rtrXrrZ_convert_to_placer rxr{ryrzr|rcrrr6catch_warningsfilterwarnings UserWarningrr)rlrr`rrbrarr;rr<rd s>    zLayer._to_implrcCsptj}tj}tj|||D] }||qWdn1s*wYtjr6|S|S)z Return startup program containing initialization operations of all parameters. NOTE(dev): This is a very low level API and only for inner developer. N) rrrZ program_guardrZ_create_init_oprrZ use_pir_api)rlZstartup_programrqrr;r;r<_startup_program s    zLayer._startup_programreLayer | Sequence[Layer] | Nonecsfdurgnttrgntttfrtn tdtjdfdd}||S)a Casts all floating point parameters and buffers to ``float`` data type. Parameters: excluded_layers(nn.Layer|list|tuple|None, optional): Specify the layers that need to be kept original data type. if excluded_layers is None, casts all floating point parameters and buffers. Default: None. Returns: Layer: self Examples: .. code-block:: python >>> import paddle >>> class Model(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self.linear = paddle.nn.Linear(1, 1) ... self.dropout = paddle.nn.Dropout(p=0.5) ... ... def forward(self, input): ... out = self.linear(input) ... out = self.dropout(out) ... return out ... >>> model = Model() >>> model.float() Model( (linear): Linear(in_features=1, out_features=1, dtype=paddle.float32) (dropout): Dropout(p=0.5, axis=None, mode=upscale_in_train) ) N9excluded_layers should be type nn.Layer or list, but got rct|tjdSrs)rfrrr9rer;r< layer_transS z Layer.float..layer_trans)rtrcrurvrrrrlrerr;rr<float$ s$    z Layer.floatctjdurtd|Sdurtjgnttr!gntt t fr-t n t dtj dfdd}| |S)a Casts all floating point parameters and buffers to ``float16`` data type. .. note:: ``nn.BatchNorm`` does not support ``bfloat16`` weights, so it would not be converted by default. Parameters: excluded_layers(nn.Layer|list|tuple|None, optional): Specify the layers that need to be kept original data type. if excluded_layers is None, casts all floating point parameters and buffers except ``nn.BatchNorm``. Default: None. Returns: Layer: self Examples: .. code-block:: python >>> # doctest: +SKIP('Paddle compiled by the user does not support float16, so keep original data type.') >>> import paddle >>> class Model(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self.linear = paddle.nn.Linear(1, 1) ... self.dropout = paddle.nn.Dropout(p=0.5) ... ... def forward(self, input): ... out = self.linear(input) ... out = self.dropout(out) ... return out ... >>> model = Model() >>> model.float16() Model( (linear): Linear(in_features=1, out_features=1, dtype=float32) (dropout): Dropout(p=0.5, axis=None, mode=upscale_in_train) ) FzQPaddle compiled by the user does not support float16, so keep original data type.Nrrcrrs)rfrrrrr;r<r rz"Layer.float16..layer_trans)rampZis_float16_supportedr6r7r BatchNormrtrcrurvrrrrr;rr<rX s *    z Layer.float16cr)a/ Casts all floating point parameters and buffers to ``bfloat16`` data type. .. note:: ``nn.BatchNorm`` does not support ``bfloat16`` weights, so it would not be converted by default. Parameters: excluded_layers(nn.Layer|list|tuple|None, optional): Specify the layers that need to be kept original data type. if excluded_layers is None, casts all floating point parameters and buffers except ``nn.BatchNorm``. Default: None. Returns: Layer: self Examples: .. code-block:: python >>> # doctest: +SKIP('bfloat need V100 compile') >>> import paddle >>> class Model(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... self.linear = paddle.nn.Linear(1, 1) ... self.dropout = paddle.nn.Dropout(p=0.5) ... ... def forward(self, input): ... out = self.linear(input) ... out = self.dropout(out) ... return out ... >>> model = Model() >>> model.bfloat16() >>> #UserWarning: Paddle compiled by the user does not support bfloat16, so keep original data type. Model( (linear): Linear(in_features=1, out_features=1, dtype=float32) (dropout): Dropout(p=0.5, axis=None, mode=upscale_in_train) ) FzRPaddle compiled by the user does not support bfloat16, so keep original data type.Nrrcrrs)rfrrrrr;r<r rz#Layer.bfloat16..layer_trans)rrZis_bfloat16_supportedr6r7r rrtrcrurvrrrrr;rr<r s +    zLayer.bfloat16)Nr)rrr`r*rr)rr)rrrr)rrX)rrrr)rrrrrr)NNFNN)rr-rrr`rrrrrrrrr )NNN)rOrrrr`rrr )T)rbrrrrs)r`rrr)rr)rr)F)rrrr)rTT)rrXrbrrrrr)rFNT) rrXrrrrrrrr)rOrXrr rrrr)rrrr)rrrrrr)r:rrrrr)r:rrr)rOrXr r/rr/)rOrXr,r rr )rr=)r?r=rr)rOrXrr)rOrXrDrrr)rOrXrr)rrO)rrZrr)NTr)r\r]rbrr^rXrr_)NTrFTT)r\r]rbrr^rXrdrrerrfrrr_)NTrTT) r\r]rbrr^rXrerrfrrr_)r)r^rXrr#)rnr_rprrrq)rrr`rrrrr) rrrrr`rrrrbrrr) r|r rrr`rrrrr )NNNTF) rrr`rrrrbrrar)rr)rerrr)>rrrrrrjrrrrrrrr'rrrrrrrrrrrrrrr&r(r#r*r+r2r<r>r@rCrJrNrSrTrYr[r`rhrmrnrorZdeprecate_stat_dictrrrrrdrZset_dict load_dictrrrr;r;r;r<r/qs 6 + : 6 : 7C >: 8 $ P  ( N J E% B  +  .K <    n    # @ + * &  N D G 5 C)^ __future__rr~rrretypingr6r collectionsrrrrrnumpyrZtyping_extensionsrrr r r Zpaddle.autograd.backward_utilsr Z paddle.baser rrZpaddle.base.corerZpaddle.base.dygraphrZpaddle.base.dygraph.baserrrrZpaddle.base.dygraph_utilsrZpaddle.base.executorrrZpaddle.base.frameworkrrrrrrrrr r!Zpaddle.base.layer_helper_baser"Z5paddle.distributed.flex_checkpoint.dcp.sharded_weightr#r$Zpaddle.frameworkr%Zpaddle.profiler.utilsr&Z paddle.utilsr'collections.abcr(r)Zpaddle._typingr*r+r,r-Zpaddle.nn.initializerr.__all__rrr9rXr_rZcompilerKrMr=rGrJrQr_rfrgrrr/r;r;r;r<sh       ,           )