o
    )iG                  
   @   s   d dl mZmZ d dlZd dlZddlmZmZ ddlm	Z	m
Z
 dejdeejgejf dejjd	ed
df
ddZG dd dejjZG dd dejjZdS )    )CallableListN   )copy_to_model_parallel_region!reduce_from_model_parallel_region) sequence_parallel_leading_matmul!sequence_parallel_trailing_matmulweightinit_methodprocess_grouppartition_dimreturnc           
      C   s   |  }| }| j\}}|dkr%| || |}||d |d d f }	n| ||| }|d d |d |f }	|| t  | |	 W d    d S 1 sQw   Y  d S )Nr   )ranksizeshapeZ	new_emptytorchZno_gradZcopy_)
r	   r
   r   r   r   Z
world_sizeZnrowsZncolsZfull_weightZmy_weight_slice r   f/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/xformers/ops/modpar_layers.py_init_2d_weight   s   


"r   c                       s   e Zd Zddejjjddddedee dej	j
deded	eejgejf d
ededdf fddZdejdeej fddZ  ZS )ColumnParallelLinearTF)biasgather_outputr
   sequence_parallelfuse_sequence_parallelin_featuresout_featuresr   r   r   r
   r   r   r   Nc          
         s   t t|   t|tstd|rtd|rtd | _|| _|| _	|| _
|| _| tfdd|D s<J fdd|D | _tj fdd| jD | _| jD ]
}	t|	||d	d
 qXd S )NzTxFormers's implementation of ColumnParallelLinear requires out_features to be a listzExFormers's implementation of ColumnParallelLinear requires bias=FalsezNxFormers's implementation of ColumnParallelLinear requires gather_output=Falsec                 3   s    | ]	}|  d kV  qdS )r   Nr   .0dimmp_sizer   r   	<genexpr>Q   s    z0ColumnParallelLinear.__init__.<locals>.<genexpr>c                    s   g | ]}|  qS r   r   r   r   r   r   
<listcomp>R       z1ColumnParallelLinear.__init__.<locals>.<listcomp>c                    s"   g | ]}t jt | fqS r   )r   nn	Parameteremptyr   )r   r   r   r"   U   s    r   r   )superr   __init__
isinstancelist	TypeError
ValueErrorr   Zglobal_out_featuresr   r   r   r   allZmy_out_featuresr   r$   ZParameterListweightsr   )
selfr   r   r   r   r   r
   r   r   w	__class__)r   r    r   r)   .   s:   


zColumnParallelLinear.__init__input_c                    sN   | j rt dd | jD | j| jd}|S t | j  fdd| jD }|S )Nc                 S   s   g | ]}|  qS r   )tr   r1   r   r   r   r"   b   r#   z0ColumnParallelLinear.forward.<locals>.<listcomp>Zfuser   c                    s   g | ]
}t  | qS r   )r   matmulr5   r6   r4   r   r   r"   h   s    )r   r   r/   r   r   r   )r0   r4   Zoutputsr   r9   r   forward^   s   	zColumnParallelLinear.forward)__name__
__module____qualname__r   r$   initxavier_normal_intr   distributedProcessGroupboolr   Tensorr)   r:   __classcell__r   r   r2   r   r   -   s8    "0r   c                       sz   e Zd Zddejjjddddededejj	de
de
d	eejgejf d
e
de
f fddZdejdejfddZ  ZS )RowParallelLinearTF)r   input_is_parallelr
   r   r   r   r   r   r   rG   r
   r   r   c          
         s   t t|   |rtd|std|| _|| _|| _|| _|| _|	 }	||	 dks.J ||	 | _
tjt|| j
f| _t| j||dd d S )NzBxFormers's implementation of RowParallelLinear requires bias=FalsezNxFormers's implementation of RowParallelLinear requires input_is_parallel=Truer   r   r'   )r(   rF   r)   r-   Zglobal_in_featuresr   r   r   r   r   Zmy_in_featuresr   r$   r%   r&   r	   r   )
r0   r   r   r   r   rG   r
   r   r   r    r2   r   r   r)   m   s*   
zRowParallelLinear.__init__r4   r   c                 C   sF   | j rt|| j | j| jd}|S t|| j }t|| j}|S )Nr7   )	r   r   r	   r5   r   r   r   r8   r   )r0   r4   outputr   r   r   r:      s   	zRowParallelLinear.forward)r;   r<   r=   r   r$   r>   r?   r@   rA   rB   rC   r   rD   r)   r:   rE   r   r   r2   r   rF   l   s4    (rF   )typingr   r   r   Ztorch.distributedZdifferentiable_collectivesr   r   Zseqparr   r   rD   rA   rB   r@   r   r$   Moduler   rF   r   r   r   r   <module>   s$   
?