Ë
    ‡\;j},  ã                   óN  — d dl Z d dlZd dlZd dlZd dlmZ d dlmZ ddlm	Z	m
Z
mZ ddlmZmZmZmZmZmZmZmZmZmZmZmZ g Zdad„ Zi ad Zi ai ada g d	¢Z!da"da# e jH                  d
¬«      Z%d a&d„ Z'd„ Z(d„ Z)d„ Z*d„ Z+d„ Z,d„ Z-d„ Z.d„ Z/d„ Z0	 dd„Z1da2d„ Z3dde%fd„Z4d„ Z5d„ Z6y)é    N)Úcore)Úin_dynamic_modeé   )ÚGroupÚ_add_new_groupÚis_initialized)Ú	_c_concatÚ_c_identityÚ_c_lookup_tableÚ_c_softmax_with_cross_entropyÚ_c_splitÚ_LinearÚ_linearÚ_mp_allreduceÚ_parallel_embeddingÚ_parallel_linearÚ_set_var_distributedÚsplitc                  óV   — t         st        j                  j                  «       a t         S ©N)Ú_global_envÚpaddleÚdistributedÚParallelEnv© ó    úfG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/distributed/collective.pyÚ_get_global_envr   .   s   € åÜ×(Ñ(×4Ñ4Ó6ˆÜÐr   Ú_default_pg)ÚncclÚglooÚheterÚxcclÚbkcli  )Úsecondsc                 ó   — | a y r   )Ú_default_backend)Úbackends    r   Ú_set_default_backendr)   L   s   € àÑr   c                 ó   — | a y r   )Ú_default_store)Ústores    r   Ú_set_default_storer-   Q   s   € à�Nr   c            	      ó¶   — t         t        vrHt        «       } t        | j                  dt        t        | j                  «      «      «      t        t         <   t        S )Nr   )Ú_global_env_gidÚ
_group_mapr   r   ÚrankÚlistÚrangeÚ
world_size)Úgenvs    r   Ú_get_group_mapr6   V   sB   € äœjÑ(ÜÓ ˆÜ&+Ø�I‰I�qœ$œu T§_¡_Ó5Ó6ó'
Œ
”?Ñ#ô Ðr   c                  ó$   — t        «       t           S r   )r6   r/   r   r   r   Ú_get_global_groupr8   `   s   € ÜÓœOÑ,Ð,r   c                  ó   — t         S r   ©Ú_group_map_by_namer   r   r   Ú_get_group_map_by_namer<   d   s   € äÐr   c                  óF   — t        «       sJ d«       ‚t        «       t           S )NzZCall paddle.distributed.init_parallel_env first to initialize the distributed environment.)r   r<   Ú_default_group_namer   r   r   Ú_get_default_groupr?   i   s+   € äÔð ð	5óÐô "Ó#Ô$7Ñ8Ð8r   c                 ó*   — | t         vsJ ‚|t         | <   y r   )r0   )ÚgidÚgroups     r   Ú_set_group_maprC   r   s   € à”jÑ Ð Ð Ø„Jˆs‚Or   c                 ó*   — | t         vsJ ‚|t         | <   y r   r:   )ÚnamerB   s     r   Ú_set_group_map_by_namerF   x   s   € àÔ)Ñ)Ð)Ð)Ø$Ô�tÒr   c                 ó*   — | t         vsJ ‚|t         | <   y r   )Ú_group_map_backend)rB   r(   s     r   Ú_set_group_map_backendrI   ~   s   € àÔ*Ñ*Ð*Ð*Ø 'Ô�uÒr   c                  óÖ   — t        «       r.t        dz  at        t        t        «       j                  d«      z   S t        t        «       «      t        t        «       j                  d«      z   S )Nr   é	   )r   Ú_start_ring_idÚmaxr   ÚnringsÚlenr6   r   r   r   Ú_new_ring_idrP   „   sP   € äÔä˜!ÑˆÜ¤¤OÓ$5×$<Ñ$<¸aÓ @Ñ@Ð@ä”>Ó#Ó$¤s¬?Ó+<×+CÑ+CÀQÓ'GÑGÐGr   c                 ó²  — d }t        «       }| t        v s
J d| z  «       ‚| dk(  r$t        j                  j	                  ||||«      }|S | dk(  r/t        j
                  j	                  |||||j                  «      }|S | dk(  r/t        j                  j	                  ||j                  |||«      }|S | dk(  r"t        j                  j	                  ||||«      }|S )NzUnsupported backend: %s.r!   r    r#   r$   )
r   Ú_valid_backend_listr   ÚProcessGroupGlooÚcreateÚProcessGroupNCCLÚ
pg_timeoutÚProcessGroupCustomÚdevice_typeÚProcessGroupBKCL)	r(   r,   r1   r4   Ú
group_nameÚ
pg_optionsÚgroup_idÚpgr5   s	            r   Ú_new_process_group_implr^   Ž   sï   € ð 
€BÜÓ€DØÔ)Ñ)ÐOÐ+EÈÑ+OÓOÐ)Ø�&ÒÜ×"Ñ"×)Ñ)¨%°°zÀ8ÓLˆð €Ið 
�FÒ	Ü×"Ñ"×)Ñ)Ø�4˜ X¨t¯©ó
ˆð €Ið 
�FÒ	Ü×$Ñ$×+Ñ+Ø�4×#Ñ# T¨:°xó
ˆð
 €Ið 
�FÒ	Ü×"Ñ"×)Ñ)¨%°°zÀ8ÓLˆØ€Ir   c                 ó   — | a y r   )Ú_custom_gid)rA   s    r   Ú_set_custom_gidra   ¯   s   € à�Kr   c           	      ój  — t        «       �r}t        rt        n	t        «       }t        t	        |«      z   }|dk7  r^| �t        | «      dkD  rNt        «       }|j                  }|j                  }|€t        n|}| €|} t        | «      t        |«      k  sJ d«       ‚t        | «      }t        | «      } |dkD  r3| v r/|dk(  rdn| j                  |«      }	t        |t        |	||d|¬«      }
nd}	d}
t        |	|| |
|¬«      }|t        |<   |t         |<   |t"        |<   t%        |«       t'        t)        j*                  d	d«      «      dk(  rFt,        j.                  j1                  t-        j2                  dgt,        j4                  ¬
«      |d¬«       |S |sd}|dk(  sJ d«       ‚t7        «       }|j                  }t        «       }|| vrt        d|| «      }|t         |<   �n^t        | «      } | j                  |«      }t        | «      }t        ||| «      }|t         |<   |dk\  �rt9        j:                  «       }||_        ||_        | D �cg c]  }|j@                  |   ‘Œ c}|_         |jB                  |_!        d|_"        t9        jF                  «       rEt9        jH                  |jJ                  «      }t9        jL                  ||«      jO                  |«       nft9        jP                  «       rEt9        jR                  |jJ                  «      }t9        jT                  ||«      jO                  |«       ntW        d«      ‚|S t        «       rt-        jX                  dgd¬
«      nt-        jZ                  dgdd¬
«      }t,        j.                  j1                  |d¬«       t,        j.                  j]                  |«       |S c c}w )aù  

    Creates a new distributed communication group.

    Args:
        ranks (list): The global ranks of group members.
        backend (str): The backend used to create group, only nccl is supported now.
        timeout (datetime.timedelta, optional): The waiting timeout for store relevant options, default is 30 minutes.

    Returns:
        Group: The group instance.

    Examples:
        .. code-block:: python

            >>> # doctest: +REQUIRES(env: DISTRIBUTED)
            >>> import paddle

            >>> paddle.distributed.init_parallel_env()
            >>> tindata = paddle.randn(shape=[2, 3])
            >>> gp = paddle.distributed.new_group([2, 4, 6])
            >>> paddle.distributed.all_reduce(tindata, group=gp, sync_op=False)

    r"   Nr   zQSize of new group must be less than or equal to that of the default global group.r   )r[   r\   éÿÿÿÿ)r]   rE   Ú$FLAGS_eager_communication_connection)ÚdtypeT)rB   Úsync_opr    z,backend other than nccl is not supported yeté   zno cuda device foundÚint32)rf   )/r   r`   rP   r>   ÚstrrO   r?   r1   Úranksr'   ÚsortedÚindexr^   r+   r   r;   r0   rH   r   ÚintÚosÚgetenvr   r   Ú
all_reduceÚzerosÚuint8r   r   ÚParallelStrategyÚnranksÚ
local_rankÚtrainer_endpointsÚcurrent_endpointrN   Úis_compiled_with_cudaÚ	CUDAPlaceÚ	device_idÚNCCLParallelContextÚinit_with_ring_idÚis_compiled_with_xpuÚXPUPlaceÚBKCLParallelContextÚAssertionErrorÚ	to_tensorÚfullÚwait)rj   r(   ÚtimeoutrA   rZ   Úglobal_groupÚglobal_rankÚglobal_ranksÚsizer1   r]   rB   r5   Úring_idÚgpÚ
group_rankÚ
group_sizeÚstrategyÚiÚplaceÚtmps                        r   Ú	new_groupr‘   ´   sg  € ô6 Õå(�k¬l«nˆÜ(¬3¨s«8Ñ3ˆ
Ø�gÒ 5 =´C¸³JÀ²NÜ-Ó/ˆLØ&×+Ñ+ˆKØ'×-Ñ-ˆLØ*1¨/Õ&¸wˆGØˆ}Ø$�Ü�u“:¤ \Ó!2Ò2ð ð=óÐ2ô �5‹zˆÜ�u“ˆØ�!Š8˜ uÑ,Ø 7Ò*‘1°·±¸KÓ0HˆDÜ(ØÜØØØØØô‰Bð ˆDØˆBÜ�d˜C ¨2°JÔ?ˆØ).Ô˜:Ñ&ØŒ
�3‰Ø$+Ô˜5Ñ!ô 	�uÔäŒr�y‰yÐ?ÀÓCÓDÈÒIÜ×Ñ×)Ñ)Ü—‘˜a˜S¬¯©Ô5¸UÈDð *ô ð ˆáØˆØ�fÒÐLÐLÓLÐäÓ€DØ—)‘)€Kä‹n€Gà˜%ÑÜ�2�w Ó&ˆØ Œ
�7Óä�u“ˆØ—[‘[ Ó-ˆ
Ü˜“Zˆ
Ü�:˜w¨Ó.ˆØ Œ
�7Ñà˜‹?Ü×,Ñ,Ó.ˆHØ(ˆHŒOØ",ˆHÔá38ó*Ù38¨a�×&Ñ& qÓ)°5ñ*ˆHÔ&ð )-×(=Ñ(=ˆHÔ%ØˆHŒOä×)Ñ)Ô+ÜŸ™ t§~¡~Ó6�Ü×(Ñ(¨°5Ó9×KÑKØõô ×*Ñ*Ô,ÜŸ™ d§n¡nÓ5�Ü×(Ñ(¨°5Ó9×KÑKØõô %Ð%;Ó<Ð<àˆIô Ôô 	×Ñ˜!˜ GÕ,ä�[‰[˜!˜˜a wÔ/ð ô
 ×Ñ×!Ñ! #¨tÐ!Ô4Ü
×Ñ×Ñ˜CÔ Ø€Iùò=*s   È>N0c                  ó*   — t        j                  «       S )a!  
    Check whether the distributed package is available.

    Returns:
        Returns True if the distributed package is available, otherwise False.

    Examples:
        .. code-block:: python

            >>> import paddle
            >>> print(paddle.distributed.is_available())

    )r   Úis_compiled_with_distr   r   r   Úis_availabler”   5  s   € ô ×%Ñ%Ó'Ð'r   c                 ó&  — t        j                  «       }t        «       }|j                  }|j                  }|j
                  }| dk(  r#t         j                  j                  |d||«       y | dk(  r¢d}|j                  D ]  }||z  }Œ	 |dj                  d«      z  }t        j                  |j                  d¬«      «      j                  «       }t         j                  j                  |«       t         j                  j                  |d|||«       y | dk(  rT|j                   }	t"        j$                  j'                  |	› d	|› �«       t         j                  j)                  |d|||	«       y | d
k(  r¢d}|j                  D ]  }||z  }Œ	 |dj                  d«      z  }t        j                  |j                  d¬«      «      j                  «       }t         j                  j                  |«       t         j                  j+                  |d|||«       y y )Nr!   Ú0r    Ú z
ring_id:{}zUTF-8)Úencodingr#   Ú:r$   )r   Úcreate_or_get_global_tcp_storer   r1   r4   rz   ÚCommContextManagerÚcreate_gloo_comm_contextrv   ÚformatÚhashlibÚmd5ÚencodeÚ	hexdigestÚset_device_idÚcreate_nccl_comm_contextrX   r   ÚdeviceÚ
set_deviceÚcreate_xccl_comm_contextÚcreate_bkcl_comm_context)
r(   r,   Ú
global_envr1   r4   Údev_idÚendpoints_strÚendpointÚendpoints_str_hashÚdev_types
             r   Ú_init_parallel_envr®   F  sì  € Ü×/Ñ/Ó1€EÜ Ó"€JØ�?‰?€DØ×&Ñ&€JØ×!Ñ!€Fà�&ÒÜ×Ñ×8Ñ8Ø�3˜˜jõ	
ð 
�FÒ	ØˆØ"×4Ô4ˆHØ˜XÑ%‰Mð 5à˜×,Ñ,¨SÓ1Ñ1ˆÜ$Ÿ[™[Ø× Ñ ¨'Ð Ó2ó
ç
‰)‹+ð 	ô 	×Ñ×-Ñ-¨fÔ5Ü×Ñ×8Ñ8Ø�3˜˜jÐ*<õ	
ð 
�FÒ	Ø×)Ñ)ˆÜ�‰× Ñ  H :¨Q¨v¨hÐ!7Ô8Ü×Ñ×8Ñ8Ø�3˜˜j¨(õ	
ð 
�FÒ	ØˆØ"×4Ô4ˆHØ˜XÑ%‰Mð 5à˜×,Ñ,¨SÓ1Ñ1ˆÜ$Ÿ[™[Ø× Ñ ¨'Ð Ó2ó
ç
‰)‹+ð 	ô 	×Ñ×-Ñ-¨fÔ5Ü×Ñ×8Ñ8Ø�3˜˜jÐ*<õ	
ð 
r   )r   )7Údatetimerž   rn   r   Úpaddle.baser   Úpaddle.frameworkr   Úcommunication.groupr   r   r   Úfleet.layers.mpu.mp_opsr	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__r   r   r0   r/   r;   rH   r>   rR   r+   r'   Ú	timedeltaÚ_default_timeoutrL   r)   r-   r6   r8   r<   r?   rC   rF   rI   rP   r^   r`   ra   r‘   r”   r®   r   r   r   Ú<module>r·      s÷   ðó Û Û 	ã õ Ý ,ç FÑ F÷÷ ÷ ó ð €à€òð €
Ø€ð Ð ð Ð ð $Ð â?Ð Ø€ØÐ Ø%�8×%Ñ%¨dÔ3Ð Ø€òò
ò
ò-òò
9òò%ò(òHð" óð< €òð
  $Ð0@ó ~òB(ó"(
r   