o * i*5@sUddlmZddlZddlZddlmZmZddlmZddl Z ddl m Z ddl m Z ddlmZmZmZdd lmZmZmZmZmZmZmZmZmZmZmZmZmZereed Z d e!d <dd l"m#Z#gZ$da%ddZ&ia'dZ(ia)ia*da+gdZ,da-da.ej/ddZ0da1ddZ2ddZ3ddZ4ddZ5ddZ6ddZ7d d!Z8d"d#Z9d$d%Z:d&d'Z; dNd(d)Zdde0ddfdOd8d9Z?dPd;d<Z@dQd>d?ZAiaBd@dAZCdBdCZDdDdEZEdFdGZFdRdSdJdKZGdRdSdLdMZHdS)T) annotationsN) TYPE_CHECKINGLiteral) TypeAlias)core)in_dynamic_mode)Group_add_new_groupis_initialized) _c_concat _c_identity_c_lookup_table_c_softmax_with_cross_entropy)_c_softmax_with_multi_label_cross_entropy_c_split_Linear_linear _mp_allreduce_parallel_embedding_parallel_linear_set_var_distributedsplit)glooncclxcclbkclflagcxr _BackendList) NCCLConfigcCststjatSN) _global_envpaddle distributedZ ParallelEnvr$r$i/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/paddle/distributed/collective.py_get_global_env:s r&Z _default_pg)rrheterrrri)secondscC|adSr )_default_backend)backendr$r$r%_set_default_backendXr,cCr)r )_default_store)storer$r$r%_set_default_store]r-r0cCs.ttvrt}t|jdtt|jtt<tS)Nr)_global_env_gid _group_mapr&r ranklistrange world_sizegenvr$r$r%_get_group_mapbs r9cCs ttSr )r9r1r$r$r$r%_get_global_groupls r:cCtSr _group_map_by_namer$r$r$r%_get_group_map_by_namepr>cCstsJdttS)NzZCall paddle.distributed.init_parallel_env first to initialize the distributed environment.)r r>_default_group_namer$r$r$r%_get_default_groupus rAcC|tvsJ|t|<dSr )r2)gidgroupr$r$r%_set_group_map~  rEcCrBr r<)namerDr$r$r%_set_group_map_by_namerFrHcCrBr )_group_map_backend)rDr+r$r$r%_set_group_map_backendrFrJcCs8trtd7atttjdSttttjdS)Nr )r_start_ring_idmaxr&nringslenr9r$r$r$r% _new_ring_idsrPc Csd} t} |tvsJd|d|dkr tj||||} | S|dkr3tj||||| j||} | S|dkrDtj|| j|||} | S|dkrStj ||||} | S|dkrctj ||||| j|} | S)NzUnsupported backend: .rrrrr) r&_valid_backend_listrZProcessGroupGloocreateZProcessGroupNCCLZ pg_timeoutZProcessGroupCustom device_typeZProcessGroupBKCLZProcessGroupFlagcx) r+r/r3r6 group_name pg_optionsgroup_idnccl_comm_init_option nccl_configpgr8r$r$r%_new_process_group_implsF   r[cCr)r ) _custom_gid)rCr$r$r%_set_custom_gidr-r]rankslist[int] | Noner+Literal['nccl'] | Nonetimeoutdatetime.timedeltarXintrYNCCLConfig | Nonereturnr c sptrtrtnt}tt|}|dkrA|dust|dkrAt}|j}|j} |dur-t n|}|dur5| }t|t| ksAJdt|} t |}| dkrj||vrj|dkrWdn| |} t |t | | |d|||d } nd} d} t| ||| |d} | t|<| t|<|t| <t| | S|sd }|d ksJd tj}t}||vrtd||}|t|<nft |}| |}t|}t|||}|t|<|d krt}||_||_fd d |D|_j|_d|_trtj}t|||nt r t!j}t"|||nt#d|Strt$j%dgddnt$j&dgddd}t$j'j(|ddt$j')||S)a Creates a new distributed communication group. Args: ranks (list): The global ranks of group members. backend (str): The backend used to create group, only nccl is supported now. timeout (datetime.timedelta, optional): The waiting timeout for store relevant options, default is 30 minutes. Returns: Group: The group instance. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env: DISTRIBUTED) >>> import paddle >>> paddle.distributed.init_parallel_env() >>> tindata = paddle.randn(shape=[2, 3]) >>> gp = paddle.distributed.new_group([2, 4, 6]) >>> paddle.distributed.all_reduce(tindata, group=gp, sync_op=False) r'NrzQSize of new group must be less than or equal to that of the default global group.r)rVrWrXrY)rZrGrz,backend other than nccl is not supported yetcsg|]}j|qSr$)trainer_endpoints).0ir7r$r% 3s znew_group..zno cuda device foundZint32)ZdtypeT)Zsync_op)*rr\rPr@strrOrAr3r^r*sortedindexr[r.r r=r2rIr r&rZParallelStrategyZnranksZ local_rankrhZcurrent_endpointrNZis_compiled_with_cudaZ CUDAPlace device_idZNCCLParallelContextZinit_with_ring_idZis_compiled_with_xpuZXPUPlaceZBKCLParallelContextAssertionErrorr"Z to_tensorfullr#Z all_reducewait)r^r+rarXrYrCrUZ global_groupZ global_rankZ global_rankssizer3rZrDZring_idgpZ group_rankZ group_sizeZstrategyZplacetmpr$r7r% new_groups!              rvboolcCstS)a! Check whether the distributed package is available. Returns: Returns True if the distributed package is available, otherwise False. Examples: .. code-block:: python >>> import paddle >>> print(paddle.distributed.is_available()) )rZis_compiled_with_distr$r$r$r% is_availableTsrxNonec CsNt}t}|j}|j}|j}|dkrtj|d||dS|dkrSd}|jD]}||7}q(|d d7}t |j dd }tj|tj|d|||dS|dkrq|j} tj| d |tj|d||| dS|d krd}|jD]}||7}qz|d d7}t |j dd }tj|tj|d|||dSdS) Nr0rz ring_id:{}zUTF-8)encodingr:r)rZcreate_or_get_global_tcp_storer&r3r6roZCommContextManagerZcreate_gloo_comm_contextrhformathashlibmd5encode hexdigestZ set_device_idZcreate_nccl_comm_contextrTr"ZdeviceZ set_deviceZcreate_xccl_comm_contextZcreate_bkcl_comm_context) r+r/Z global_envr3r6Zdev_idZ endpoints_strZendpointZendpoints_str_hashZdev_typer$r$r%_init_parallel_envesX           rcCr;r _shutdown_group_map_by_namer$r$r$r%_get_shutdown_group_map_by_namer?rcCs |t|<dSr r)pg_namerDr$r$r%"_update_shutdown_group_map_by_name rcCs t|=dSr r)rr$r$r%"_delete_shutdown_group_map_by_names rcCs tdSr )rclearr$r$r$r%!_clear_shutdown_group_map_by_namerrrD Group | NonecCst}|dur*tD]\}}|jdur'||vr'|tkr'|jt||q dS|jdurA|j|vrC|jt|j|dSdSdSr )rr>items process_groupr@shutdownrrG)rDshutdown_groupsrrZr$r$r%shutdown_process_groups      rcCsdt}|dur|D]}|jq tdS|jdur.|j|vr0|jt|jdSdSdSr )rvaluesrZrestartrrGr)rDrrZr$r$r%restart_process_groups    r)rrN) r^r_r+r`rarbrXrcrYrdrer )rerw)r+rreryr )rDrrery)I __future__rdatetimertypingrrZtyping_extensionsrr"Z paddle.baserZpaddle.frameworkrZcommunication.groupr r r Zfleet.layers.mpu.mp_opsr r rrrrrrrrrrrr__annotations__Zpaddle.base.libpaddler__all__r!r&r2r1r=rIr@rRr.r* timedeltaZ_default_timeoutrLr,r0r9r:r>rArErHrJrPr[r\r]rvrxrrrrrrrrr$r$r$r%st    <      /   +