o )ij@sXUddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z m Z m Z ddlZddlmZmZmZmZeGdddeZeGdd d eZeGd d d eZeGd d d eZde eeeffddZeZdddeDZeGdddeZeGdddeZdefddZ dlddZ!dmdej"fddZ#dmdej"fdd Z$dmdej"fd!d"Z%dmdej"fd#d$Z&dmdej"fd%d&Z'dej"fd'd(Z(ej)d)d*Z*d+d,Z+iej,j-j.e+ej,j-j/e+ej,j-j0e(ej,j-j1e!ej,j-j2e!ej,j-j3e!ej,j-j4ee!d-d.ej,j-j5e!ej,j-j6ee!d/d.ej,j-j7ee!d-d.ej,j-j8ee!d/d.ej,j-j9e#ej,j-j:e#ej,j-j;e&ej,j-jej,j-j.e+ej,j-j/e+ej,j-j0e(ej,j-j;e&ej,j-jZJejKjLd?dd@gdAd8e ed9ej"d:ej"d;e ej"d:sr-c@$eZdZeddZdZdeZdS)Sp24GemmCuspltSearchatenZ_cslt_sparse_mm_searchrz_cslt_sparse_mm_search@Nrrrr rr_cusplt_version_strrrrrrr/= r/c@r.)Sp24GemmCuspltr0Z_cslt_sparse_mmrz_cslt_sparse_mm@Nr1rrrrr4Dr3r4cCsBtdk}|sdSd}tjrtjd}tdkr|dkrdS|S)N)rrF)rrcuda)rr) r)_cusplt_versionr"r6r%Zget_device_capability) availableZcompute_capabilityrrr_has_cusparseLtKs  r;rc Cs8d}|D] }t|tr |}q|dusJg}t|D]]\}}t|tjrrt|tsG||vr3t||}ntd|jd|jd|dt ||j duse|j duse|j |j kse|j |j krrtd|jd|jd| |qt|tsJdt|j|dd|D|j|d d|D|j|j S) Nz Operation r(zL on Sparse24Tensor requires all operands to be Sparse24Tensors, but operand z is a z] on Sparse24Tensor requires all operands to be Sparse24Tensors with the same sparsity patternz$Only implemented for CUTLASS tensorscS g|] }t|tr |jn|qSr) isinstanceSparse24Tensorpackedr+xrrr ~s z)sparse24_pointwise_op..cSr<r)r=r>packed_tr@rrrrBs)r=r> enumerater"Tensorsparsify24_like ValueErrorrrtype threads_masksZdata_ptrstrideappendSparse24TensorCutlassshapemetameta_t) functypesargskwargsallow_sparsify_args_listselftensorZ args_updatedirrrsparse24_pointwise_opZs\         rXcCstt|dksJ|\}}|jdks|jdkrtdt|tr$||S|}t|ts/J|j|ddS)N8`Sparse24Tensor` matmul: Broadcasting is not implementedT)prefer_col_major_output)lenndimNotImplementedErrorr=r>_mmt)rPrQrRrSABB_trrr sparse24_mms  rdcCst|dksJ|\}}}|jdks|jdkrtd|jdkr(td|jt|tr1td|}t|tsr`r_)rPrQrRrSrgrarbrcrrrsparse24_addmms"    rhcCsdt|dvsJ|dd\}}t|dkr|dnd}|dur&||Stdd|||gdS)N)rYrerYre)rPrQrR)r\r`rh)rPrQrRrSrarbrgrrrsparse24_linears  ric Cspt|dksJ|d}t|tsJt|jdksJ|j|jd|jdf|j|j|j|j|j dddS)Nr rrYr?rNrCrOrI) r\r=r>rM __class__rCrOr?rNrIZ transposerPrQrRrSrUrrr sparse24_ts rncCs:t|dksJ|\}}t||jkrtd|d|S)NrYzO`view` is not implemented for Sparse24Tensor, except for the dummy case (shape=))r\tuplerMr^)rPrQrRrSrUrMrrr sparse24_views rqc Cs<t|dksJ|d}|j|j|j|j|j|j|jddS)Nr rF)rMr?rNrCrOrI requires_grad)r\rlrMr?rNrCrOrIrmrrrsparse24_detachsrsccs tj}zdVW~dS~wr))r"_CZ_DisableTorchDispatch)guardrrr no_dispatchs   rvcCs2t ||WdS1swYdSr))rv)rPrQrRrSrrrfallback_dispatchers$rw)rr )rT)rc @seZdZUejed<ejed<ejed<ejed<ejed<gdZedddejdejdejdejdejf d d Zd d Z d ejfddZ ddddejde de ejd ejfddZ ejjZddZeddZdS)r>r?rNrCrOrIrkFrrc CsLt|tjsJtjj|||j|j|d}||_||_||_||_ ||_ |S)N)devicedtyperr) r=r"rEZ_make_wrapper_subclassryrzr?rNrCrOrI) clsrMr?rNrCrOrIrrrVrrr__new__+s zSparse24Tensor.__new__cCs|jjd|jdS)Nz(shape=ro)rlrrMrUrrr__repr__FszSparse24Tensor.__repr__r cCs*tj|jd|jd|j|jd}||S)Nr )ryrz)r"eyerMryrz)rUerrr_sp24_to_denseIszSparse24Tensor._sp24_to_denseNr[rgrbr[rgcCstr))r^)rUrbr[rgrrrr_QszSparse24Tensor._mmcCs|j|j|jffSr)) __slots__rMrrr}rrr__tensor_flatten__\z!Sparse24Tensor.__tensor_flatten__cCs |\}}||fi|d|iS)Nrrr)r{Z inner_tensorsZ flatten_specZ outer_sizeZ outer_striderMrrrrr__tensor_unflatten___sz#Sparse24Tensor.__tensor_unflatten__)rrrr"rE__annotations__r staticmethodr|r~rboolrr_rtZ_disabled_torch_function_implZ__torch_function__r classmethodrrrrrr>!sJ          r>c @sFeZdZddddejdeejdedejfdd Zed d d Z dS)rLNFrfrbrgr[r c Cst|tr td|durtdtdtd|jdks"|jdkr,td|jjd|j d|j d krXtd|jjd |j d d |j dd |j d d |j dd t |j ||j d|j d S)NB`Sparse24Tensor @ Sparse24Tensor` is not supported by the hardwarez`Sparse24Tensor` with backend='zF' does not support matmul with bias. Remove the bias, or use backend=''rY`)` matmul: Broadcasting is not implementedr r` matmul: invalid shapes (, ) @ (ro)r=r>rGr^BACKEND_CUTLASSBACKEND_CUSPARSELTr]rlrrMrrr?rN)rUrbrgr[rrrr_ls6   zSparse24TensorCutlass._mmrcC6|jtvrt|jd|jdt|j||||SNzI only supports a specific set of operations, can't perform requested op (ro)_overloadpacketSPARSE24_DISPATCH_CUTLASSr^rr{rPrQrRrSrrr__torch_dispatch__ z(Sparse24TensorCutlass.__torch_dispatch__rN) rrrr"rErrr_rrrrrrrLks rL_CUSPLT_ALG_CACHEZXFORMERS_CUSPARSELT_TUNE01rMr?rbrgtranspose_resultc Cs.tsdS|\}}|jd}d}||ddkrdnd7}||r!dnd7}|||||j|duf} | tvr7t| Sd} g} tdD]G} d } t| D](}z tj||||| d Wn tyad } Ynw|dkrot j t }qG| rtnt j t |}| || fq?| | ddt| <t| S) a cuSPARSELt has multiple algorithms (that correspond to different kernels) to run a given GEMM, because the optimal kernel depends on the GEMM dimensions. This function attempts to find the most efficient one by benchmarking all of them. NOTE: cuSPARSELt also provides a function to search the best algorithm (exposed via `aten:_cslt_sparse_mm_search`) but it often fails to find the best algorithm, so we need this workaround. rr rrjcN FFrgralg_idT) _CUSPLT_TUNErMrJrzrranger4r RuntimeErrorr"r6Z synchronizetime monotonicrKsort)rMr?rbrgrMKNfmthREPEATZ TIME_ALGOalgoZ has_errorrWr`dtrrr_cusplt_find_algsF        rzxformers::_cusplt_mmr6)Z mutates_argsZ device_typescCs&t|||||d}tj|||||dS)z This operator wraps find_algo + gemm. This is because we don't want find_algo to be visible by torch compile, otherwise it will remove it from the graph. rgrr)rr4r)rMr?rbrgrrrrr _cusplt_mms   rcCsF|\}}|jd}|rtj||g|j|jdStj||g|j|jdS)Nr )rzry)rMr"emptyrzry)rMr?rbrgrrrrrrr_cusplt_mm_metas  rc @sFeZdZddddejdedeejdejfdd Zed d d Z dS)Sparse24TensorCuSparseLtFNrrbr[rgr c Cst|tr td|jdks|jdkrtd|jjd|jd|jdkrItd|jjd|jdd|jdd |jdd|jdd |jdd dkrhtd|jjd t|jd t|jd|j |j krtd|jjd t|jd t|jd|j d|j d |dur|j |j krtd|jjd t|jd t|jd|j d|j d t sJt j j j|j|j|||d}|r|}|d|jdS)NrrYrrr rrrrroz` matmul: trying to do `A=z @ B=zD`. The dense matrix B should have the second dimension aligned to 8.z`, with A.dtype=z and B.dtype=zH. This operation is only supported when A and B have the same data type.z + C`, with A.dtype=B.dtype=z and C.dtype=zK. This operation is only supported when A, B and C have the same data type.r)r=r>rGr]r^rlrrMrprzr;r"opsZxformersrr?r`)rUrbr[rgoutrrrr_sf  & $$ zSparse24TensorCuSparseLt._mmrcCrr)rSPARSE24_DISPATCH_CUSPARSELTr^rrrrrr#rz+Sparse24TensorCuSparseLt.__torch_dispatch__r) rrrr"rErrr_rrrrrrrs 1rZ24sparseZ24denseZsteZcutlassr$ZdenserArbackendc Cs||ttfvs Jd|t|tr|jdurtd|Stj|||d\}}}}}|tkr0tnt }||j |||||ddS)NInvalid backend: z'Input to `sparsify24` is already sparse) algorithmrF)r?rNrCrOrIrr) rrr=r>rIrGrrrLrrM) rArrr?rNrCrOrIr{rrr_sparsify24_forward;s4  rc @s@eZdZedejdededefddZedejfdd Zd S) _Sparsify24FuncrArgradientrc Csf|tttfvrtd|dtdtdtt|||d}|j|_|j|_|j|_|j|_||_ |S)NzInvalid gradient type: 'z '. Expected 'z' or 'rr) GRADIENT_SP24GRADIENT_DENSE GRADIENT_STErGrrIrNrOrzr)ctxrArrrrrrrforwardYs"z_Sparsify24Func.forwardgrad_outc Cst|ts |jtkr|dddfSt|trJ|j|jksJ|jtkr@t||j\}}}}t |j ||j ||j |j|j d}n|jtkrT|jsLJt||j}nJd|j|dddfS)NrxFzUnsupported gradient type: )r=r>rrrzrrrrIrLrMrNrOrrr is_contiguousr)rrr?_rCgrad_inrrrbackwardhs0   z_Sparsify24Func.backwardN) rrrrr"rEr*rrrrrrrXs rc @sDeZdZedejdedededef ddZedejfd d Z d S) _Sparsify24STEFuncrArrbw_mul0bw_mul1cCs&t|||d}|j|_||_||_|S)Nr)rrIrr)rrArrrrrrrrrs  z_Sparsify24STEFunc.forwardrcCsNt|trJ|jdkr|jdkr|}n tj||j|j|jd}|ddddfS)N?)Zmul0Zmul1)r=r>rrrrrI)rrrrrrrsz_Sparsify24STEFunc.backwardN) rrrrr"rEr*floatrrrrrrrs rc @s@eZdZedejdededefddZedejfdd Z d S) _Sparsify24LikeFuncrApatternrrc Cst|ts td|jstd|tttfvr!td|d|j|_|j |_ |j |_ |j |_ ||_ |t krF|js?Jt||jStj||j|d\}}}}|tkrgt|j||j ||j |j|jdS|tksrJd|||j ||j t|j|||||j|jdS)Nz4`sparsify24_like`: `pattern` must be a sparse tensorzA`sparsify24_like` is not implemented when `pattern` is transposedz*`sparsify24_like`: invalid gradient type ""rrxr)r=r>r^rIrrrrrGrNrOrzr BACKEND_DENSErrrrrLrMrrrZcopy_r) rrArrrr?rNrCrOrrrrsV    z_Sparsify24LikeFunc.forwardrc Cs|jtks t|tr|dddfSt|trJ|j|jksJ|jtkr6|js+Jt ||jdddfS|jt ks=Jt j ||jt d\}}}}t |j||j||j|j|jddddfS)Nrrx)rrr=r>rzrrIrrrrrrrLrMrNrOrr)rrr?rrCrrrrs8    z_Sparsify24LikeFunc.backwardN) rrrrr"rEr>r*rrrrrrrs -rF.)boundrPcCstttj|Sr))rrr"_dynamoallow_in_graph)rPrrrrrrrcCst||||Sr))rapply)rArrrrrr sparsify24srrrrcCst|||||S)a  2:4 sparsification, with Straight Through Estimator for the backward pass (eg the gradient is *not* sparsified). Optionally, `bw_mul[0-1]` provide the option to rescale the gradient differently for pruned (`bw_mul0`) and kept values (`bw_mul1`). )rr)rArrrrrrrsparsify24_stesrr out_densecCs||dur|rt}|dkrt|trtnt}t|ts#tdt||j s6t | | || St ||||S)Nrz/`pattern` must be a `Sparse24Tensor` but got a ) rr=rrrr>rGrHrIrrrr`)rArrrrrrrrF s    rF)rNrr)d contextlibosr functoolsrtypingrrrrrrr r r"commonr r rrrrrrintr'r9joinr2r/r4rr;rXrErdrhrirnrqrscontextmanagerrvrwrr0Z is_same_sizeZdetach_detachZreluZgeluZsilumuladdZ gelu_backwardZ silu_backwardZthreshold_backwardmmmatmulr`viewZlinearrZaddmmrr>rLrr*rzrenvirongetrrZlibraryZ custom_oprZ register_fakerrrrrrrrrrrZautogradFunctionrrrrrrrrFrrrrs  (  2                    $         J&( 4   > ."V