o p if@sxddlmZddlZddlmZddlmZmZddlm Z m Z ddl Z ddl mZddlZddlmZmZmZmZddlmZmZdd lmZdd lmZmZmZdd lmZdd lm Z m!Z!dd l"m#Z$m%Z&ddl'm(Z(ddl)m*Z*ddl+m,Z,e rddlm-Z-ddlm.Z.ddl/m0Z0m1Z1m2Z2m3Z3m4Z4e-dZ5e-dZ6e-dZ7gZ8    d]d^d(d)Z9Gd*d+d+Z:d_d/d0Z;d`d2d3Z<    d]d4d5Z=    d]d6d7Z>   dadbd>> import paddle >>> inputs = paddle.rand((4, 23, 16)) >>> prev_h = paddle.randn((4, 32)) >>> cell = paddle.nn.SimpleRNNCell(16, 32) >>> rnn = paddle.nn.RNN(cell) >>> outputs, final_states = rnn(inputs, prev_h) >>> print(outputs.shape) [4, 23, 32] >>> print(final_states.shape) [4, 32] )r _rnn_dynamic_graph_rnn_static_graph)r.r0r1r3r5r7r8r=Z/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/paddle/nn/layer/rnn.pyrnn@s*A r?c@s*eZdZdddZddd Zdd d ZdS) ArrayWrapperxrr9NonecCs |g|_dSN)arrayselfrAr=r=r>__init__ zArrayWrapper.__init__rcCs|j||SrC)rDappendrEr=r=r>rIs zArrayWrapper.appenditemintcCs |j|SrC)rD __getitem__)rFrJr=r=r>rLrHzArrayWrapper.__getitem__N)rArr9rB)rArr9r)rJrKr9r)__name__ __module__ __qualname__rGrIrLr=r=r=r>r@s  r@state new_state step_maskcCs0tjjj||ddtjjj|d|dd}|S)z3update rnn state or just pass the old state throughraxisr)paddletensormathZ_multiply_with_axis)rPrQrRr=r=r> _maybe_copys rXrAcCs(ddgttdt|j}t||S)Nrr)listrangelenshaperU transpose)rApermr=r=r>_transpose_batch_times r`c s|rdndtj|}|dj}|dur"|j||rdndd}|s+tjt|}|durCtjjj j |||j d} t | ddg} |r\tjdd|}|durZtj | dgdnd} |} g} t|D]Ctjfdd|} || | fi|\} }|durtjtt| d | |}|} dkrtjd d| n tjd d| | } qdtjfd d| }|rtjfd d|}|}||fS)Nrr batch_ref batch_dim_idxmaxlendtypecStj|dgdSNrrSrUreverserAr=r=r>z$_rnn_dynamic_graph..rScs|SrCr=rk)ir=r>rlrRcSst|SrC)r@rkr=r=r>rlrocSs ||SrC)rI)rAZx_arrayr=r=r>rls cstj|jdSNrS)rUstackrDrktime_step_indexr=r>rlrmcstj|dSrqrirkrsr=r>rl)rUutilsflattenr]get_initial_states map_structurer`staticnn sequence_lod sequence_maskrfr^rjr[rrX)r.r0r1r3r5r7r8Z flat_inputsZ time_stepsmaskstatesoutputsZ step_inputsZ step_outputs new_states final_outputs final_statesr=)rnrtr>r;s^       r;c st|dttttjjfdt|ttfr,t|D]\}}t |dt |dddgdqt|dtttt dtjjfdt|dtt dtjjfdd!d d } |dur^|j ||rZd nd d}tj | |}|sntj t|}ttj |d d } t| tj} |durtjjjj|| tj |d jd} t| d d g} |rtj dd|}|durtj| d gdnd} tjjdtjgdd| } t| d} | k} Wdn1swYtjjj | }tj!j"tj |d jd}tj dd|}tj fdd|||#|}tj fdd|}|||fi|\}}t|tjjjtjjfs _switch_gradsz'_rnn_static_graph.._switch_gradrrrardcSrgrhrirkr=r=r>rl+rmz#_rnn_static_graph..rScpuZint64rfcSstjj|jdS)Nr)rUrV create_arrayrfrkr=r=r>rl@rmctj||SrCrUrV array_writerAystart_ir=r>rlDrmcstj|SrC)rUrVZ array_readrkrr=r>rlMrucs||dS)N?r=rrpr=r>rl[s)rAvaluecrrCrrrr=r>rlermTrTZ use_stackcSst|ddddS)NrTrrrkr=r=r>rlqscSs|dSNr=rkr=r=r>rlurocSrgrhrirkr=r=r>rlyrmF)+r rrZtuplerUpirValue isinstance enumeraterstrtyperxrvryr`r]rwcastZint32rzr{r|r}rfr^rjbaser Z device_guardZzerosZ control_flowWhilerVrblockZassert_same_structureZ unsqueezer increment less_thanZassignr)r.r0r1r3r5r7r8rnZinput_xrZ max_seq_lenr~endZcondZwhile_opZ out_arrayZ init_arrayZstep_inZ pre_staterrZnew_condout_Z all_staterrr=)rrRr>r<s             %r<cell_fwcell_bw+tuple[Tensor, Tensor] | list[Tensor] | None$tuple[Tensor, tuple[Tensor, Tensor]]c Ks|dur|j||r dndd}|j||rdndd}n|\}}t||||fd|i|\} }t||||f|dd|\} }tjdd | | } ||f} | | fS) a birnn creates a bidirectional recurrent neural network specified by RNNCell `cell_fw` and `cell_bw`, which performs :code:`cell.call()` (for dygraph mode :code:`cell.forward`) repeatedly until reaches to the maximum length of `inputs` and then concat the outputs for both RNNs along the last axis. Parameters: cell_fw(RNNCellBase): An instance of `RNNCellBase`. cell_bw(RNNCellBase): An instance of `RNNCellBase`. inputs(Tensor): the input sequences. If time_major is True, the shape is `[time_steps, batch_size, input_size]` else the shape is `[batch_size, time_steps, input_size]`. initial_states(tuple|None, optional): A tuple of initial states of `cell_fw` and `cell_bw`. If not provided, `cell.get_initial_states` would be called to produce initial state for each cell. Defaults to None. sequence_length (Tensor|None, optional): shape `[batch_size]`, dtype: int64 or int32. The valid lengths of input sequences. Defaults to None. If `sequence_length` is not None, the inputs are treated as padded sequences. In each input sequence, elements whose time step index are not less than the valid length are treated as paddings. time_major (bool): Whether the first dimension of the input means the time steps. Defaults to False. **kwargs: Additional keyword arguments to pass to `forward` of each cell. Returns: outputs (Tensor): the outputs of the bidirectional RNN. It is the concatenation of the outputs from the forward RNN and backward RNN along the last axis. If time_major is True, the shape is `[time_steps, batch_size, size]`, else the shape is `[batch_size, time_steps, size]`, where size is `cell_fw.hidden_size + cell_bw.hidden_size`. final_states (tuple): A tuple of the final states of the forward cell and backward cell. Examples: .. code-block:: python >>> import paddle >>> cell_fw = paddle.nn.LSTMCell(16, 32) >>> cell_bw = paddle.nn.LSTMCell(16, 32) >>> rnn = paddle.nn.BiRNN(cell_fw, cell_bw) >>> inputs = paddle.rand((2, 23, 16)) >>> outputs, final_states = rnn(inputs) >>> print(outputs.shape) [2, 23, 64] >>> print(final_states[0][0].shape) [2, 32] Nrrrar5T)r5r7cSst||gdSr)rUconcatrr=r=r>rlrmzbirnn..)rxr?rUrvry) rrr0r1r3r5r8Z states_fwZ states_bwZ outputs_fwZ outputs_bwrrr=r=r>birnnsD@      rrr#r&state_componentsrK!list[Tensor] | list[list[Tensor]]cCs|dkrt|}|s |Stt|ddd|dddSt||ks&Jtdd|D}|s7tt|Stt|}tt|ddd|dddS)a Split states of RNN network into possibly nested list or tuple of states of each RNN cells of the RNN network. Parameters: states (Tensor|tuple|list): the concatenated states for RNN network. When `state_components` is 1, states in a Tensor with shape `(L*D, N, C)` where `L` is the number of layers of the RNN network, `D` is the number of directions of the RNN network(1 for unidirectional RNNs and 2 for bidirectional RNNs), `N` is the batch size of the input to the RNN network, `C` is the hidden size of the RNN network. When `state_components` is larger than 1, `states` is a tuple of `state_components` Tensors that meet the requirements described above. For SimpleRNNs and GRUs, `state_components` is 1, and for LSTMs, `state_components` is 2. bidirectional (bool): whether the state is of a bidirectional RNN network. Defaults to False. state_components (int): the number of the components of the states. see `states` above. Defaults to 1. Returns: A nested list or tuple of RNN cell states. If `bidirectional` is True, it can be indexed twice to get an RNN cell state. The first index indicates the layer, the second index indicates the direction. If `bidirectional` is False, it can be indexed once to get an RNN cell state. The index indicates the layer. Note that if `state_components` is larger than 1, an RNN cell state can be indexed one more time to get a tensor of shape(N, C), where `N` is the batch size of the input to the RNN cell, and `C` is the hidden size of the RNN cell. rNrYcSg|]}t|qSr=)rUunstack.0rJr=r=r> z split_states..)rUrrZzipr\r)rr&rr=r=r> split_statess) "  "rSequence[Tensor]Tensor | tuple[Tensor, Tensor]cCs^|dkr ttj|Stj|}g}t|D] }|||d|qtdd|DS)a Concatenate a possibly nested list or tuple of RNN cell states into a compact form. Parameters: states (list|tuple): a possibly nested list or tuple of RNN cell states. If `bidirectional` is True, it can be indexed twice to get an RNN cell state. The first index indicates the layer, the second index indicates the direction. If `bidirectional` is False, it can be indexed once to get an RNN cell state. The index indicates the layer. Note that if `state_components` is larger than 1, an RNN cell state can be indexed one more time to get a tensor of shape(N, C), where `N` is the batch size of the input to the RNN cell, and `C` is the hidden size of the RNN cell. bidirectional (bool): whether the state is of a bidirectional RNN network. Defaults to False. state_components (int): the number of the components of the states. see `states` above. Defaults to 1. Returns: Concatenated states for RNN network. When `state_components` is 1, states in a Tensor with shape `(L\*D, N, C)` where `L` is the number of layers of the RNN network, `D` is the number of directions of the RNN network(1 for unidirectional RNNs and 2 for bidirectional RNNs), `N` is the batch size of the input to the RNN network, `C` is the hidden size of the RNN network. rNcSrr=)rUrrrr=r=r>rLrz!concat_states..)rUrrrvrwr[rIr)rr&r componentsrnr=r=r> concat_states!s$  rc@s@eZdZdZ    ddddZedddZedddZdS)r/z RNNCellBase is the base class for abstraction representing the calculations mapping the input and state to the output and new state. It is suitable to and mostly used in RNN. Nrrbrr]!NestedStructure[ShapeLike] | Nonerf!NestedStructure[DTypeLike] | None init_valuefloatrcrKr9NestedStructure[Tensor]c stj|d}dd}Gddd|dur|jn|}tjjj}|tjj_tjfdd|}|tjj_z dur>|jn} Wn tyNt } Ynwt tj| d krltj| dtjfd d|} |} |j |dkrt | tr| dD] } |j || j d<q~nXt | tr| D] } |j || j d<qnE|j || j d<nrlruzLRNNCellBase.get_initial_states.._is_shape_sequence..TF)rrZrrdictrr)seqr=r=r>_is_shape_sequences  z:RNNCellBase.get_initial_states.._is_shape_sequencec@seZdZddZdS)z-RNNCellBase.get_initial_states..ShapecSs.|ddkr t||_dSdgt||_dS)Nrr)rZr])rFr]r=r=r>rGs z6RNNCellBase.get_initial_states..Shape.__init__N)rMrNrOrGr=r=r=r>Shapes rNcs|SrCr=r])rr=r>rlroz0RNNCellBase.get_initial_states..rcsSrCr=rrr=r>rlscstj|j|dS)Nr]Z fill_valuerf)rUfullr])r]rf)rr=r>rls )rUrvrw state_shapeZ layers_utilsZ is_sequencery state_dtypeNotImplementedErrorr Zget_default_dtyper\r]rrZrrJ) rFrbr]rfrrcrZ states_shapesZis_sequence_oriZ states_dtypesZ fill_shapessZ init_statesr=)rrfrr>rxVsd'                zRNNCellBase.get_initial_statesrBcCtd)a Abstract method (property). Used to initialize states. A (possibly nested structure of) shape[s], where a shape is a list/tuple of integers (-1 for batch size would be automatically inserted into a shape if shape is not started with it). Not necessary to be implemented if states are not initialized by `get_initial_states` or the `shape` argument is provided when using `get_initial_states`. z=Please add implementation for `state_shape` in the used cell.rrFr=r=r>r zRNNCellBase.state_shapecCr)a Abstract method (property). Used to initialize states. A (possibly nested structure of) data types[s]. The structure must be same as that of `shape`, except when all tensors' in states has the same data type, a single data type can be used. Not necessary to be implemented if states are not initialized by `get_initial_states` or the `dtype` argument is provided when using `get_initial_states`. z=Please add implementation for `state_dtype` in the used cell.rrr=r=r>rrzRNNCellBase.state_dtype)NNrr) rbrr]rrfrrrrcrKr9rr9rB)rMrNrO__doc__rxpropertyrrr=r=r=r>r/Os  o csTeZdZdZ      d d!fdd Zd"d#ddZed$ddZd%ddZZ S)& SimpleRNNCella Elman RNN (SimpleRNN) cell. Given the inputs and previous states, it computes the outputs and updates states. The formula used is as follows: .. math:: h_{t} & = act(W_{ih}x_{t} + b_{ih} + W_{hh}h_{t-1} + b_{hh}) y_{t} & = h_{t} where :math:`act` is for :attr:`activation`. Please refer to `Finding Structure in Time `_ for more details. Parameters: input_size (int): The input size. hidden_size (int): The hidden size. activation (str, optional): The activation in the SimpleRNN cell. It can be `tanh` or `relu`. Defaults to `tanh`. weight_ih_attr (ParamAttr|None, optional): The parameter attribute for :math:`weight_ih`. Default: None. weight_hh_attr(ParamAttr|None, optional): The parameter attribute for :math:`weight_hh`. Default: None. bias_ih_attr (ParamAttr|None, optional): The parameter attribute for the :math:`bias_ih`. Default: None. bias_hh_attr (ParamAttr|None, optional): The parameter attribute for the :math:`bias_hh`. Default: None. name (str|None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Variables: - **weight_ih** (Parameter): shape (hidden_size, input_size), input to hidden weight, corresponding to :math:`W_{ih}` in the formula. - **weight_hh** (Parameter): shape (hidden_size, hidden_size), hidden to hidden weight, corresponding to :math:`W_{hh}` in the formula. - **bias_ih** (Parameter): shape (hidden_size, ), input to hidden bias, corresponding to :math:`b_{ih}` in the formula. - **bias_hh** (Parameter): shape (hidden_size, ), hidden to hidden bias, corresponding to :math:`b_{hh}` in the formula. Inputs: - **inputs** (Tensor): shape `[batch_size, input_size]`, the input, corresponding to :math:`x_{t}` in the formula. - **states** (Tensor, optional): shape `[batch_size, hidden_size]`, the previous hidden state, corresponding to :math:`h_{t-1}` in the formula. When states is None, zero state is used. Defaults to None. Returns: - **outputs** (Tensor): shape `[batch_size, hidden_size]`, the output, corresponding to :math:`h_{t}` in the formula. - **states** (Tensor): shape `[batch_size, hidden_size]`, the new hidden state, corresponding to :math:`h_{t}` in the formula. Notes: All the weights and bias are initialized with `Uniform(-std, std)` by default. Where std = :math:`\frac{1}{\sqrt{hidden\_size}}`. For more information about parameter initialization, please refer to :ref:`api_paddle_ParamAttr`. Examples: .. code-block:: python >>> import paddle >>> x = paddle.randn((4, 16)) >>> prev_h = paddle.randn((4, 32)) >>> cell = paddle.nn.SimpleRNNCell(16, 32) >>> y, h = cell(x, prev_h) >>> print(y.shape) [4, 32] r,N input_sizerK hidden_size activation_ActivationType | strweight_ih_attrParamAttrLike | Noneweight_hh_attr bias_ih_attr bias_hh_attrname str | Noner9rBc st|dkrtd|jjd|dt|} |dur1|j||f|t | | d|_ n|j||fdt dd|_ d|j _ |durX|j||f|t | | d|_ n|j||fdt dd|_ d|j _ |dur|j|f|dt | | d|_n|j|fddt d d|_d|j_ |dur|j|f|dt | | d|_n|j|fddt d d|_d|j_ ||_||_|d vrtd |||_|d krtj|_dStj|_dS) Nrhidden_size of + must be greater than 0, but now equals to rFdefault_initializerTZis_biasrrr+z=activation for SimpleRNNCell should be tanh or relu, but get r,)superrG ValueError __class__rMrWsqrtcreate_parameterIUniform weight_ihConstantr weight_hhbias_ihbias_hhrrrrUr,Fr-_activation_fn) rFrrrrrrrrstdrr=r>rG(s          zSimpleRNNCell.__init__r0rrr4cCs||dur |||j}|}tj||jdd}|jdur ||j7}tj||jdd}|jdur3||j7}|||}||fS)NTZ transpose_y) rxrrUmatmulrrrrr)rFr0rZpre_hZi2hZh2hhr=r=r>r$s    zSimpleRNNCell.forward tuple[int]cC|jfSrCrrr=r=r>rszSimpleRNNCell.state_shapercCs(d}|jdkr |d7}|jdi|jS)N{input_size}, {hidden_size}r,z, activation={activation}r=)rformat__dict__)rFrr=r=r> extra_reprs zSimpleRNNCell.extra_repr)r,NNNNN)rrKrrKrrrrrrrrrrrrr9rBrC)r0rrr4r9rr9r rMrNrOrrGr$rrr __classcell__r=r=rr>rsE W rcsTeZdZdZ      dd fdd Zd!d"ddZed#ddZd$ddZZ S)%LSTMCella Long-Short Term Memory(LSTM) RNN cell. Given the inputs and previous states, it computes the outputs and updates states. The formula used is as follows: .. math:: i_{t} & = \sigma(W_{ii}x_{t} + b_{ii} + W_{hi}h_{t-1} + b_{hi}) f_{t} & = \sigma(W_{if}x_{t} + b_{if} + W_{hf}h_{t-1} + b_{hf}) o_{t} & = \sigma(W_{io}x_{t} + b_{io} + W_{ho}h_{t-1} + b_{ho}) \widetilde{c}_{t} & = \tanh (W_{ig}x_{t} + b_{ig} + W_{hg}h_{t-1} + b_{hg}) c_{t} & = f_{t} * c_{t-1} + i_{t} * \widetilde{c}_{t} h_{t} & = o_{t} * \tanh(c_{t}) y_{t} & = h_{t} If `proj_size` is specified, the dimension of hidden state :math:`h_{t}` will be projected to `proj_size`: .. math:: h_{t} = h_{t}W_{proj\_size} where :math:`\sigma` is the sigmoid function, and * is the elementwise multiplication operator. Please refer to `An Empirical Exploration of Recurrent Network Architectures `_ for more details. Parameters: input_size (int): The input size. hidden_size (int): The hidden size. weight_ih_attr(ParamAttr|None, optional): The parameter attribute for `weight_ih`. Default: None. weight_hh_attr(ParamAttr|None, optional): The parameter attribute for `weight_hh`. Default: None. bias_ih_attr (ParamAttr|None, optional): The parameter attribute for the `bias_ih`. Default: None. bias_hh_attr (ParamAttr|None, optional): The parameter attribute for the `bias_hh`. Default: None. proj_size (int, optional): If specified, the output hidden state will be projected to `proj_size`. `proj_size` must be smaller than `hidden_size`. Default: None. name (str|None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Variables: - **weight_ih** (Parameter): shape (4 * hidden_size, input_size), input to hidden weight, which corresponds to the concatenation of :math:`W_{ii}, W_{if}, W_{ig}, W_{io}` in the formula. - **weight_hh** (Parameter): shape (4 * hidden_size, hidden_size), hidden to hidden weight, which corresponds to the concatenation of :math:`W_{hi}, W_{hf}, W_{hg}, W_{ho}` in the formula. If proj_size was specified, the shape will be (4 * hidden_size, proj_size). - **weight_ho** (Parameter, optional): shape (hidden_size, proj_size), project the hidden state. - **bias_ih** (Parameter): shape (4 * hidden_size, ), input to hidden bias, which corresponds to the concatenation of :math:`b_{ii}, b_{if}, b_{ig}, b_{io}` in the formula. - **bias_hh** (Parameter): shape (4 * hidden_size, ), hidden to hidden bias, which corresponds to the concatenation of :math:`b_{hi}, b_{hf}, b_{hg}, b_{ho}` in the formula. Inputs: - **inputs** (Tensor): shape `[batch_size, input_size]`, the input, corresponding to :math:`x_t` in the formula. - **states** (list|tuple, optional): a list/tuple of two tensors, each of shape `[batch_size, hidden_size]`, the previous hidden state, corresponding to :math:`h_{t-1}, c_{t-1}` in the formula. When states is None, zero state is used. Defaults to None. Returns: - **outputs** (Tensor). Shape `[batch_size, hidden_size]`, the output, corresponding to :math:`h_{t}` in the formula. If `proj_size` is specified, output shape will be `[batch_size, proj_size]`. - **states** (tuple). A tuple of two tensors, each of shape `[batch_size, hidden_size]`, the new hidden states, corresponding to :math:`h_{t}, c_{t}` in the formula. If `proj_size` is specified, shape of :math:`h_{t}` will be `[batch_size, proj_size]`. Notes: All the weights and bias are initialized with `Uniform(-std, std)` by default. Where std = :math:`\frac{1}{\sqrt{hidden\_size}}`. For more information about parameter initialization, please refer to :ref:`api_paddle_ParamAttr`. Examples: .. code-block:: python >>> import paddle >>> x = paddle.randn((4, 16)) >>> prev_h = paddle.randn((4, 32)) >>> prev_c = paddle.randn((4, 32)) >>> cell = paddle.nn.LSTMCell(16, 32) >>> y, (h, c) = cell(x, (prev_h, prev_c)) >>> print(y.shape) [4, 32] >>> print(h.shape) [4, 32] >>> print(c.shape) [4, 32] NrrrKrrrrrr proj_sizerrr9rBc st|dkrtd|jjd||dkr%td|jjd|||kr-tddt|} |durK|jd||f|t | | d |_ n|jd||fdt dd |_ d |j _ |durx|jd||pk|f|t | | d |_ n|jd||p|fdt dd |_ d |j _ |dur|jd|f|d t | | d |_n|jd|fdd t d d |_d |j_ |dur|jd|f|d t | | d |_n|jd|fdd t d d |_d |j_ ||_|dkr|j||f|t | | d |_||_||_tj|_tj|_dS) Nrrrz proj_size of z*proj_size must be smaller than hidden_sizerFrTrr)rrGrrrMrWrrrrrrrrrrr weight_horrrsigmoid_gate_activationrUr, _activation) rFrrrrrrrrrrr=r>rGs             zLSTMCell.__init__r0rrSequence[Tensor] | Nonec Cs|dur |||j}|\}}tj||jdd}|jdur"||j}|tj||jdd7}|jdur7||j}tj|ddd}| |d}| |d}| |d} |||| |d } | | | } |j dkrtt| |j } | | | ffS) NTrrrZnum_or_sectionsrTrrrY) rxrrUrrrrrsplitr r rr) rFr0r pre_hiddenZpre_cellZgatesZ chunked_gatesrnfocrr=r=r>r$Ts$      zLSTMCell.forwardtuple[tuple[int], tuple[int]]cCs|jf|jp|jffS)a The `state_shape` of LSTMCell is a tuple with two shapes: `((hidden_size, ), (hidden_size,))`. (-1 for batch size would be automatically inserted into shape). These two shapes correspond to :math:`h_{t-1}` and :math:`c_{t-1}` separately. )rrrr=r=r>rkszLSTMCell.state_shapercCdjdi|jSNrr=rrrr=r=r>ruzLSTMCell.extra_repr)NNNNrN)rrKrrKrrrrrrrrrrKrrr9rBrC)r0rrr )r9rrrr=r=rr>rsa _  rcsTeZdZdZ     ddfdd Z d d!ddZed"ddZd#ddZZ S)$GRUCellaj Gated Recurrent Unit (GRU) RNN cell. Given the inputs and previous states, it computes the outputs and updates states. The formula for GRU used is as follows: .. math:: r_{t} & = \sigma(W_{ir}x_{t} + b_{ir} + W_{hr}h_{t-1} + b_{hr}) z_{t} & = \sigma(W_{iz}x_{t} + b_{iz} + W_{hz}h_{t-1} + b_{hz}) \widetilde{h}_{t} & = \tanh(W_{ic}x_{t} + b_{ic} + r_{t} * (W_{hc}h_{t-1} + b_{hc})) h_{t} & = z_{t} * h_{t-1} + (1 - z_{t}) * \widetilde{h}_{t} y_{t} & = h_{t} where :math:`\sigma` is the sigmoid function, and * is the elementwise multiplication operator. Please refer to `An Empirical Exploration of Recurrent Network Architectures `_ for more details. Parameters: input_size (int): The input size. hidden_size (int): The hidden size. weight_ih_attr(ParamAttr|None, optional): The parameter attribute for `weight_ih`. Default: None. weight_hh_attr(ParamAttr|None, optional): The parameter attribute for `weight_hh`. Default: None. bias_ih_attr (ParamAttr|None, optional): The parameter attribute for the `bias_ih`. Default: None. bias_hh_attr (ParamAttr|None, optional): The parameter attribute for the `bias_hh`. Default: None. name (str|None, optional): Name for the operation (optional, default is None). For more information, please refer to :ref:`api_guide_Name`. Variables: - **weight_ih** (Parameter): shape (3 * hidden_size, input_size), input to hidden weight, which corresponds to the concatenation of :math:`W_{ir}, W_{iz}, W_{ic}` in the formula. - **weight_hh** (Parameter): shape (3 * hidden_size, hidden_size), hidden to hidden weight, which corresponds to the concatenation of :math:`W_{hr}, W_{hz}, W_{hc}` in the formula. - **bias_ih** (Parameter): shape (3 * hidden_size, ), input to hidden bias, which corresponds to the concatenation of :math:`b_{ir}, b_{iz}, b_{ic}` in the formula. - **bias_hh** (Parameter): shape (3 * hidden_size, ), hidden to hidden bias, which corresponds to the concatenation of :math:`b_{hr}, b_{hz}, b_{hc}` in the formula. Inputs: - **inputs** (Tensor): A tensor with shape `[batch_size, input_size]`, corresponding to :math:`x_t` in the formula. - **states** (Tensor): A tensor with shape `[batch_size, hidden_size]`, corresponding to :math:`h_{t-1}` in the formula. Returns: - **outputs** (Tensor): shape `[batch_size, hidden_size]`, the output, corresponding to :math:`h_{t}` in the formula. - **states** (Tensor): shape `[batch_size, hidden_size]`, the new hidden state, corresponding to :math:`h_{t}` in the formula. Notes: All the weights and bias are initialized with `Uniform(-std, std)` by default. Where std = :math:`\frac{1}{\sqrt{hidden\_size}}`. For more information about parameter initialization, please refer to s:ref:`api_paddle_ParamAttr`. Examples: .. code-block:: python >>> import paddle >>> x = paddle.randn((4, 16)) >>> prev_h = paddle.randn((4, 32)) >>> cell = paddle.nn.GRUCell(16, 32) >>> y, h = cell(x, prev_h) >>> print(y.shape) [4, 32] >>> print(h.shape) [4, 32] NrrKrrrrrrrrr9rBc st|dkrtd|jjd|dt|}|dur3|jd||f|t | |d|_ n|jd||fdt dd|_ d|j _ |dur^|jd||f|t | |d|_ n|jd||fdt dd|_ d|j _ |dur|jd|f|dt | |d |_n|jd|fddt d d |_d|j_ |dur|jd|f|dt | |d |_n|jd|fddt d d |_d|j_ ||_||_tj|_tj|_dS) NrrrrFr rTrr)rrGrrrMrWrrrrrrrrrrrrrrr rUr,r ) rFrrrrrrrrrr=r>rGs|              zGRUCell.__init__r0rrr4tuple[Tensor, Tensor]cCs|dur |||j}|}tj||jdd}|jdur ||j}tj||jdd}|jdur3||j}tj|ddd\}}}tj|ddd\} } } | || } | || } | || | }||| |}||fS)NTrr rr ) rxrrUrrrrrrr r )rFr0rrZx_gatesZh_gatesZx_rZx_zZx_cZh_rZh_zZh_crzrrr=r=r>r$s     zGRUCell.forwardrcCr)z The `state_shape` of GRUCell is a shape `[hidden_size]` (-1 for batch size would be automatically inserted into shape). The shape corresponds to the shape of :math:`h_{t-1}`. rrr=r=r>r/szGRUCell.state_shapercCrrrrr=r=r>r8rzGRUCell.extra_repr)NNNNN)rrKrrKrrrrrrrrrrr9rBrC)r0rrr4r9rrrrr=r=rr>rysQQ  rcs8eZdZdZ  ddfd d Z ddddZZS)RNNa Wrapper for RNN, which creates a recurrent neural network with an RNN cell. It performs :code:`cell.forward()` repeatedly until reaches to the maximum length of `inputs`. Parameters: cell(RNNCellBase): An instance of `RNNCellBase`. is_reverse (bool, optional): Indicate whether to calculate in the reverse order of input sequences. Defaults to False. time_major (bool): Whether the first dimension of the input means the time steps. Defaults to False. Inputs: - **inputs** (Tensor): A (possibly nested structure of) tensor[s]. The input sequences. If time_major is False, the shape is `[batch_size, time_steps, input_size]`. If time_major is True, the shape is `[time_steps, batch_size, input_size]` where `input_size` is the input size of the cell. - **initial_states** (Tensor|list|tuple, optional): Tensor of a possibly nested structure of tensors, representing the initial state for the rnn cell. If not provided, `cell.get_initial_states` would be called to produce the initial states. Defaults to None. - **sequence_length** (Tensor, optional): shape `[batch_size]`, dtype: int64 or int32. The valid lengths of input sequences. Defaults to None.If `sequence_length` is not None, the inputs are treated as padded sequences. In each input sequence, elements whose time step index are not less than the valid length are treated as paddings. - **kwargs**: Additional keyword arguments to pass to `forward` of the cell. Outputs: - **outputs** (Tensor|list|tuple): the output sequences. If `time_major` is True, the shape is `[time_steps, batch_size, hidden_size]`, else `[batch_size, time_steps, hidden_size]`. - **final_states** (Tensor|list|tuple): final states of the cell. Tensor or a possibly nested structure of tensors which has the same structure with initial state. Each tensor in final states has the same shape and dtype as the corresponding tensor in initial states. Notes: This class is a low-level API for wrapping rnn cell into a RNN network. Users should take care of the state of the cell. If `initial_states` is passed to the `forward` method, make sure that it satisfies the requirements of the cell. Examples: .. code-block:: python >>> import paddle >>> inputs = paddle.rand((4, 23, 16)) >>> prev_h = paddle.randn((4, 32)) >>> cell = paddle.nn.SimpleRNNCell(16, 32) >>> rnn = paddle.nn.RNN(cell) >>> outputs, final_states = rnn(inputs, prev_h) >>> print(outputs.shape) [4, 23, 32] >>> print(final_states.shape) [4, 32] Fr.r/r7r6r5r9rBcs8t||_t|jds|jj|j_||_||_dS)Ncall)rrGr.hasattrr$rr7r5)rFr.r7r5rr=r>rGms    z RNN.__init__Nr0rr1r2r3r8rcKs.t|j|f|||j|jd|\}}||fS)N)r1r3r5r7)r?r.r5r7)rFr0r1r3r8rrr=r=r>r${s  z RNN.forward)FF)r.r/r7r6r5r6r9rBNN)r0rr1r2r3rr8rrMrNrOrrGr$rr=r=rr>r<s3rcs6eZdZdZ ddfd d Z ddddZZS)BiRNNa Wrapper for bidirectional RNN, which builds a bidirectional RNN given the forward rnn cell and backward rnn cell. A BiRNN applies forward RNN and backward RNN with corresponding cells separately and concats the outputs along the last axis. Parameters: cell_fw (RNNCellBase): A RNNCellBase instance used for forward RNN. cell_bw (RNNCellBase): A RNNCellBase instance used for backward RNN. time_major (bool, optional): Whether the first dimension of the input means the time steps. Defaults to False. Inputs: - **inputs** (Tensor): the input sequences of both RNN. If time_major is True, the shape of is `[time_steps, batch_size, input_size]`, else the shape is `[batch_size, time_steps, input_size]`, where input_size is the input size of both cells. - **initial_states** (list|tuple|None, optional): A tuple/list of the initial states of the forward cell and backward cell. Defaults to None. If not provided, `cell.get_initial_states` would be called to produce the initial states for each cell. Defaults to None. - **sequence_length** (Tensor|None, optional): shape `[batch_size]`, dtype: int64 or int32. The valid lengths of input sequences. Defaults to None. If `sequence_length` is not None, the inputs are treated as padded sequences. In each input sequence, elements whose time step index are not less than the valid length are treated as paddings. - **kwargs**: Additional keyword arguments. Arguments passed to `forward` for each cell. Outputs: - **outputs** (Tensor): the outputs of the bidirectional RNN. It is the concatenation of the outputs from the forward RNN and backward RNN along the last axis. If time_major is True, the shape is `[time_steps, batch_size, size]`, else the shape is `[batch_size, time_steps, size]`, where size is `cell_fw.hidden_size + cell_bw.hidden_size`. - **final_states** (tuple): A tuple of the final states of the forward cell and backward cell. Notes: This class is a low level API for wrapping rnn cells into a BiRNN network. Users should take care of the states of the cells. If `initial_states` is passed to the `forward` method, make sure that it satisfies the requirements of the cells. Examples: .. code-block:: python >>> import paddle >>> cell_fw = paddle.nn.LSTMCell(16, 32) >>> cell_bw = paddle.nn.LSTMCell(16, 32) >>> rnn = paddle.nn.BiRNN(cell_fw, cell_bw) >>> inputs = paddle.rand((2, 23, 16)) >>> outputs, final_states = rnn(inputs) >>> print(outputs.shape) [2, 23, 64] >>> print(final_states[0][0].shape,len(final_states),len(final_states[0])) [2, 32] 2 2 Frr/rr5r6r9rBcsjt||_||_|j|jkrtd|jd|jd|j|jfD] }t|ds/|j|_q$||_ dS)Nzinput size of forward cell(z') does not equalsthat of backward cell()r) rrGrrrrrr$rr5)rFrrr5r.rr=r>rGs     zBiRNN.__init__Nr0rr1rr3r4r8rrcKsNt|ttfrt|dksJdt|j|j||||jfi|\}}||fS)NrYzr$s  z BiRNN.forwardr)rr/rr/r5r6r9rBr) r0rr1rr3r4r8rr9rr r=r=rr>r!s4r!csdeZdZdZ         d.d/fdd Zd0ddZd1d'd(Z  d2d1d)d*Zd3d,d-ZZ S)4RNNBasez RNNBase class for RNN networks. It provides `forward`, `flatten_parameters` and other common methods for SimpleRNN, LSTM and GRU. rr$FrNrmode_RNNType | strrrKr num_layers direction_DirectionType | strr5r6dropoutrrrrrrrr9rBc stddg} ||_||_||_||_|| vrdnd|_||_||_|dkr*dnd|_ || | | d}| |_ | dkrA|dksAJ|dkrLt }| |d<n$|d krSt }n|d kr^t }d |d <n|d krit }d|d <nt }|j|d <| ps|}|dvrd}|||fi|}|t|||td|D]}|||fi|}|t|||qnN|| vr|||fi|}|||fi|}|t|||td|D]!}|d||fi|}|d||fi|}|t|||qntd|d|_|jt||d|| vr dndkM_g}t|jD]:t|jD]1}|dkr'dnd|ddg| dur:|d| durD|dfdd|D}qqt||D] \}}t|||qY|dS)Nr&r%rYrr')rrrrrrr(r)r-rr*r,)r$FzQdirection should be forward or bidirect (or bidirectional), received direction = TrZ_reversezweight_ih_l{}{}zweight_hh_l{}{}z bias_ih_l{}{}z bias_hh_l{}{}csg|]}|qSr=)r)rrAlayersuffixr=r>rNz$RNNBase.__init__..)rrGr$rrr)num_directionsr5r&rrrrrrrIrr[r!rcould_use_cudnnr\ parametersextendrsetattrflatten_parameters)rFr$rrr&r'r5r)rrrrrZbidirectional_listr8Zrnn_clsZin_sizer7r.rrrZ param_namesnumrparamrr+r>rGs             zRNNBase.__init__cCsV|jr)|jdd}dd|D}dgt||_t|D]%\}}|ddkr)dnd|j|j}|d}||j||d|d<q|jt |g|dj t d d g|_ trvtjjjd dgd ttjjj dd dd|_d|j_n|jtjjjd td|_trt.|dj }t|tjrtjj j!|}t"#|j|j|j dddddd|  WddS1swYt$t%t%Vt8trt&#|j|dj dddd dddgg n|j'j(dd|ji|j|j ddd|dj ddWdn1s wYWddSWddS1s"wYdSdS)zn Resets parameter data pointer to address in continuous memory block for cudnn usage. F)Zinclude_sublayerscSsg|]}t|jqSr=)npprodr])rr6r=r=r>r_r.z.RNNBase.flatten_parameters..NrrYrr)r]rfrZuint8Z dropout_state)r)rfr]rrZ trainableT)rfr copy_data use_alignrfrcoalesce_tensorInput)OutputZ FusedOutput)r9r:rfrr0rattrs))r0r1r\ _all_weightsrr&r/rr7sumrfrrZ _flat_weightrrUrrrr{r_dropout_staterZcreate_variableVarDescVarTypeUINT8r Zno_gradrZDataTyperr Zpaddle_type_to_proto_typer r;rrr _helper append_op)rFparamsr]rnr6offsetZ layer_idxrfr=r=r>r4Ts         TzRNNBase.flatten_parametersr0rr1r2r3 int | None-tuple[Tensor, Tensor | tuple[Tensor, Tensor]]c s8js tjgdtr0t|j|jj j dkj j j jdj \}}}nMjj}fddtjD}jjtjjjdd}j||dj j dkj j j jj d }|||jd } jjd | |d jstj|gdn|}|t|d krt|fS|dfS)N)rrrYrYrcsg|] }jjqSr=)rF"create_variable_for_type_inferencerfrrnr0rFr=r>rs z'RNNBase._cudnn_impl..T)rfr)r<Z WeightListZPreStateZSequenceLength)Z dropout_probZ is_bidirecrrr&r$Zis_test)ZOutStateZReserveZ DropoutStater?r>r)r5rUrVr^rr r?r@rBr)r/rrr&r$trainingrFrLrfr[rrrCrDrErGr\r) rFr0r1r3rrrPZreserver?rr=rNr> _cudnn_implsf    "zRNNBase._cudnn_implc sd|jrdnd}|j|durL|jp|jg|jgf|j|jdg|j|dkr0|j|d<n t||d<t fddt |j D}nt |tj jtjjfrZ|gn|}|jrotjrh|duro||||St||jdk|j }g}t|D]$\}}|dkrtj||j|jdd}|||||\} } || | }qt||jdk|j }| |fS) Nrrrcs$g|]}tj|ddqS)rr)rUrrMdimsrfZ fill_shaper=r>rs z#RNNBase.forward..rYZupscale_in_train)rPr$)r5rfrrr&r/r]rUrJrr[rrrzrrrr0ZdeviceZis_compiled_with_rocmrQrrrr)rPrIr) rFr0r1r3Z batch_indexrrrnZ rnn_layerrZ final_stater=rRr>r$sV  zRNNBase.forwardrcCsLd}|jdkr |d7}|jdur|d7}|jdkr|d7}|jdi|jS) Nrrz, num_layers={num_layers}Fz, time_major={time_major}rz, dropout={dropout}r=)r&r5r)rr)rFZmain_strr=r=r>r;s   zRNNBase.extra_repr) rr$FrNNNNr)r$r%rrKrrKr&rKr'r(r5r6r)rrrrrrrrrrrKr9rBr)r0rr1r2r3rJr9rKrr) rMrNrOrrGr4rQr$rrr=r=rr>r#s$  c gG