Ë
    ‡\;jØ ã                   ó
  — d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZ g Z d„ Z!d„ Z"d„ Z# ee"«      Z$ ee#«      Z% G d„ d«      Z&y)é    N)Úcompiler)Úwrap_decorator)Ú_global_flagsÚin_dynamic_mode)Úapply_build_strategyé   )Útopology)ÚDistributedStrategy)ÚMetaOptimizerFactory)ÚPaddleCloudRoleMakerÚRoleMakerBase)ÚRuntimeFactory)ÚStrategyCompiler)Úmodel_parallel_random_seed)ÚloggerÚset_log_levelc                 óf  — |j                   j                  j                  «       }t        «       d   s|S t	        | di «      }|r|d   } |j
                  d   }d|j                  i}|j                   j                  }|r(|j                  rt        j                  d«       d|_        t        | |||«      S )NÚFLAGS_apply_pass_to_programÚ_pipeline_optÚsection_programÚstartup_programÚuse_cudaz‹Currently, the fuse_all_optimizer_ops pass has conflict with fuse_all_reduce_ops pass. Disable the fuse_all_optimizer_ops pass temporarily.F)Ú_user_defined_strategyÚbuild_strategyÚ_copyr   Úgetattrr   Ú_is_collectiveÚfuse_all_reduce_opsÚfuse_all_optimizer_opsr   Úwarningr   )Úmain_programr   Úconfigr   Úpipeline_optÚ
pass_attrsÚfuse_all_reduces          úgG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/distributed/fleet/fleet.pyÚapply_ir_passesr'   %   sº   € Ø×2Ñ2×AÑA×GÑGÓI€NÜ‹?Ð8Ò9ØÐä˜<¨¸"Ó=€LÙØ#Ð$5Ñ6ˆØ)×7Ñ7Ð8IÑJˆà˜f×3Ñ3Ð4€JØ×3Ñ3×GÑG€OÙ˜>×@Ò@ô 	�‰ð Zô	
ð 16ˆÔ-äØ�o ~°zóð ó    c                 ó   ‡ — ˆ fd„}|S )Nc                  óL   •— | d   }|j                   €t        d«      ‚ ‰| i |¤ŽS )Nr   z+Fleet can not find suitable runtime handler)Ú_runtime_handleÚ
ValueError©ÚargsÚkwargsÚclsÚfuncs      €r&   Ú__impl__z*_inited_runtime_handler_.<locals>.__impl__B   s4   ø€ Ø�1‰gˆà×ÑÐ&ÜÐJÓKÐKá�TÐ$˜VÑ$Ð$r(   © ©r1   r2   s   ` r&   Ú_inited_runtime_handler_r5   A   s   ø€ ô%ð €Or(   c                 ó   ‡ — ˆ fd„}|S )Nc                  ó´   •— | d   }|j                   �?|j                   j                  «       du r#t        j                  d‰j                  z  «       y  ‰| i |¤ŽS )Nr   Tz:%s() function doesn't work when use non_distributed fleet.)Ú_role_makerÚ_is_non_distributedr   r    Ú__name__r-   s      €r&   r2   z,_is_non_distributed_check_.<locals>.__impl__N   s]   ø€ Ø�1‰gˆð �O‰OÐ'Ø—‘×3Ñ3Ó5¸Ñ=ä�N‰NØLØ—=‘=ñ"ôð á�TÐ$˜VÑ$Ð$r(   r3   r4   s   ` r&   Ú_is_non_distributed_check_r;   M   s   ø€ ô%ð €Or(   c                   ó2  — e Zd ZdZd„ Z	 	 	 	 d?d„Z	 d@d„Zd„ Zd„ Zd„ Z	d	„ Z
d
i dfd„Zd
i fd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd@d„Zd„ Zd„ Zd@d„Zd„ Zd„ ZdAd„Ze e!dBd „«       «       Z"e e!dBd!„«       «       Z#d"„ Z$e e!d#„ «       «       Z%e e!d$„ «       «       Z&e e!d%„ «       «       Z'e e!d&„ «       «       Z(e e!d'„ «       «       Z)e e!d(„ «       «       Z*e e!d)„ «       «       Z+e e!g g fd*„«       «       Z,e e!	 	 	 dCd+„«       «       Z-e e!dDd,„«       «       Z.e e!d-„ «       «       Z/e e!d.„ «       «       Z0e e!	 dEd/„«       «       Z1e e!d0„ «       «       Z2e e!	 dBd1„«       «       Z3e e!dBd2„«       «       Z4dBd3„Z5d4„ Z6d5„ Z7	 dFd6„Z8d7„ Z9dGd8„Z:d9„ Z;d:„ Z<d;„ Z=	 dHd<„Z>	 dHd=„Z?	 	 	 dHd>„Z@y)IÚFleetaô  
    Unified API for distributed training of PaddlePaddle.
    Please reference the https://github.com/PaddlePaddle/PaddleFleetX for details

    Returns:
        Fleet: A Fleet instance

    Examples:
        .. code-block:: python
            :name: code-example1

            >>> # Example1: for collective training
            >>> import paddle
            >>> paddle.enable_static()
            >>> import paddle.distributed.fleet as fleet

            >>> fleet.init(is_collective=True)

            >>> strategy = fleet.DistributedStrategy()
            >>> linear = paddle.nn.Linear(10, 10)
            >>> optimizer = paddle.optimizer.SGD(learning_rate=0.001, parameters=linear.parameters())
            >>> optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)

            >>> # do distributed training

        .. code-block:: python
            :name: code-example2

            >>> # Example2: for parameter server training
            >>> import paddle
            >>> paddle.enable_static()
            >>> import paddle.distributed.fleet as fleet
            >>> strategy = fleet.DistributedStrategy()
            >>> fleet.init(strategy=strategy)

            >>> optimizer = paddle.optimizer.SGD(learning_rate=0.001)
            >>> optimizer = fleet.distributed_optimizer(optimizer)

            >>> if fleet.is_first_worker():
            ...     print("this is first worker")

            >>> print("current node index: {}".format(fleet.worker_index()))
            >>> print("total number of worker num: {}".format(fleet.worker_num()))

            >>> if fleet.is_worker():
            ...     print("this is worker")
            >>> print("worker endpoints: {}".format(fleet.worker_endpoints(to_string=True)))

            >>> print("server num: {}".format(fleet.server_num()))
            >>> print("server endpoints: {}".format(fleet.server_endpoints(to_string=True)))

            >>> if fleet.is_server():
            ...     print("this is server")
            >>> fleet.stop_worker()

    c                 ó    — d | _         d | _        d| _        d | _        d | _        i | _        t        j                  j                  d«      | _	        y )NFg        )
r8   Ústrategy_compilerr   r+   Ú_utilÚ_contextÚpaddleÚ	optimizerÚ	OptimizerÚuser_defined_optimizer©Úselfs    r&   Ú__init__zFleet.__init__ž   sI   € ØˆÔØ!%ˆÔØ#ˆÔØ#ˆÔØˆŒ
ØˆŒÜ&,×&6Ñ&6×&@Ñ&@ÀÓ&EˆÕ#r(   Nc                 óD  — ddl m} t        |«       |€
t        «       }t	        j
                  |«      | _        |€Vt        |t        «      r#|| _	        t        | j                  ¬«      | _        not        dj                  t        |«      «      «      ‚t        |t        «      r|| _        |j                  | _	        n#t        dj                  t        |«      «      «      ‚| j                  j!                  «        ddl m} |j$                  j'                  | j                  «       t)        «       | _        t-        «       rë|j/                  «       rt1        j2                  d«       nvd	t4        j6                  v rt1        j2                  d
«       n0t9        | j                  j:                  «      t4        j6                  d	<   t<        j>                  jA                  «        | j                  jB                  s7tD        jF                  €| jI                  «        | S t1        j2                  d«       | S | j                  �ri| j                  jJ                  }| jM                  «       }| jO                  «       }	|rdnd}
tQ        tS        |	«      «      }tD        jF                  €tE        jT                  «        tD        jF                  }|| _+        |jY                  d||	|
|«       | j                  jZ                  }|xs |}|du ryd}d}|r$| j                  j\                  }t_        |d   «      }|r$| j                  j`                  }t_        |d   «      }|r	|r||k(  sJ ‚|r|n|}|dkD  rB|	|z  dk(  sJ ‚d}||z  }||z  }|D �cg c]  }||z  |k(  r|‘Œ }}|jY                  d||||«       | S c c}w )a5	  
        Initialize role_maker in Fleet.

        This function is responsible for the distributed architecture
        what you want to run your code behind.

        Args:
            role_maker (RoleMakerBase, optional): A ``RoleMakerBase`` containing the configuration
                of environment variables related to distributed training.If you did not initialize
                the rolemaker by yourself, it will be automatically initialized to PaddleRoleMaker.
                The default value is None.
            is_collective (Boolean, optional): A ``Boolean`` variable determines whether the program
                runs on Collective mode or ParameterServer mode. True means the program runs on
                Collective mode, and False means running on ParameterServer mode. The default value
                is False.
            strategy (DistributedStrategy): Extra properties for distributed training.
                For details, please refer to paddle.distributed.fleet.DistributedStrategy. Default: None.
            log_level (Integer, String, optional): A ``Integer`` or ``String`` Variable determining how hight
                the logging level is. Default is "INFO".

        Returns:
            None

        Examples:
            .. code-block:: python
                :name: code-init-example1

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

            .. code-block:: python
                :name: code-init-example2

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init(is_collective=True)

            .. code-block:: python
                :name: code-init-example3

                >>> import paddle.distributed.fleet as fleet
                >>> role = fleet.PaddleCloudRoleMaker()
                >>> fleet.init(role)

            .. code-block:: python
                :name: code-init-example4

                >>> import paddle.distributed.fleet as fleet
                >>> strategy = fleet.DistributedStrategy()
                >>> fleet.init(strategy=strategy)

            .. code-block:: python
                :name: code-init-example5

                >>> import paddle.distributed.fleet as fleet
                >>> strategy = fleet.DistributedStrategy()
                >>> fleet.init(log_level = "DEBUG")

        r   )Úparallel_helperN)Úis_collectivez8`is_collective` should be instance of `bool`, but got {}z>`role_maker` should be subclass of `RoleMakerBase`, but got {}©Úfleetz6The dygraph parallel environment has been initialized.ÚFLAGS_nccl_nringsz“You have set the environment variable FLAGS_nccl_nrings outside the program, so the nccl_comm_num in DistributedStrategy will not take effect here.z=The dygraph hybrid parallel environment has been initialized.é   ÚglobalFr   Ú	mp_degreeÚtensor_parallel_degreeÚmodel)1Úpaddle.distributedrJ   r   r
   ÚcopyÚdeepcopyr   Ú
isinstanceÚboolr   r   r8   r,   ÚformatÚtyper   Ú_generate_rolerM   ÚutilÚ_set_role_makerr   r?   r   Ú_is_parallel_ctx_initializedr   r    ÚosÚenvironÚstrÚnccl_comm_numrB   ÚdistributedÚinit_parallel_envÚheter_ccl_modeÚtpÚ_HYBRID_PARALLEL_GROUPÚ_init_hybrid_parallel_envÚshardingÚworker_indexÚ
worker_numÚlistÚrangeÚ_CommunicateGroupÚ_hcgÚset_comm_groupÚtensor_parallelÚsharding_configsÚintÚtensor_parallel_configs)rG   Ú
role_makerrK   ÚstrategyÚ	log_levelrJ   rM   Úuse_shardingÚglobal_rankÚglobal_world_sizeÚglobal_ring_idÚglobal_ranksÚcgÚuse_tensor_parallelÚuse_mpÚmp_degree_shardingÚmp_degree_tensor_parallelrr   rt   rQ   Ú
mp_ring_idÚmp_rankÚmp_group_idÚidxÚmp_group_rankss                            r&   Úinitz
Fleet.init§   sž  € õB 	7ä�iÔ àÐÜ*Ó,ˆHÜ&*§m¡m°HÓ&=ˆÔ#àÐÜ˜-¬Ô.Ø&3�Ô#Ü#7Ø"&×"5Ñ"5ô$�Õ ô !ØN×UÑUÜ˜]Ó+óóð ô ˜*¤mÔ4Ø#-�Ô Ø&0×&?Ñ&?�Õ#ä ØT×[Ñ[Ü˜ZÓ(óóð ð
 	×Ñ×'Ñ'Ô)å,à�
‰
×"Ñ" 4×#3Ñ#3Ô4ä!1Ó!3ˆÔäÔØ×;Ñ;Ô=Ü—‘ØLõð
 '¬"¯*©*Ñ4Ü—N‘NðIõô 7:Ø×3Ñ3×AÑAó7”B—J‘JÐ2Ñ3ô ×"Ñ"×4Ñ4Ô6ð ×.Ñ.×=Ò=ä×,Ñ,Ð4Ø×2Ñ2Ô4ðN ˆôK —N‘NØWôðJ ˆðE × Ó Ø×6Ñ6×?Ñ?ˆLð ×+Ñ+Ó-ˆKØ $§¡Ó 1Ðá".™Q°AˆNÜ¤Ð&7Ó 8Ó9ˆLä×(Ñ(Ð0Ü×$Ñ$Ô&Ü×*Ñ*ˆBØˆDŒIØ×ÑØØØ!ØØôð #'×"=Ñ"=×"MÑ"MÐØ!Ò8Ð%8ˆFð ˜‰Øà!"ÐØ()Ð%ÙØ#'×#>Ñ#>×#OÑ#OÐ Ü%(Ð)9¸+Ñ)FÓ%GÐ"á"à×/Ñ/×GÑGð (ô -0Ø+Ð,DÑEó-Ð)ñ Ñ 3Ø)Ð-FÒFÐFÐFñ  ñ #à.ð ð ˜1Š}Ø(¨9Ñ4¸Ò9Ð9Ð9à�
Ø%¨	Ñ1�Ø)¨YÑ6�ñ  ,ó"á+˜Ø˜iÑ'¨;Ò6ò Ø+ð ð "ð
 ×!Ñ!Ø˜W i°¸^ôð ˆùò"s   Í3Nc                 ó|  — |�|j                   dk  rt        j                  d«       y t        j                  j                  «        t        j                  j                  j                  «        t        j                  «       }t        |«      D ]#  }t        j                  j                  ||¬«       Œ% t        j                  j                  j                  «        t        j                  «       }	|	|z
  |z  }
|ry t        j                  d|› d|
› d�«       |dkD  r"|
|kD  rt        j                  d|
› d	|› �«       y y y )
Nr   z)allreduce_perf is invalid, group invalid!©Úgroupz[AllReduceTest] nbytes úB test result: ú s/iteréÿÿÿÿz([Perf Warnning] AllReduce Test Timeout! ú > )Únranksr   r    rB   rc   ÚbarrierÚdeviceÚcudaÚsynchronizeÚtimerm   Ú
all_reduceÚinfo)rG   Ú	iterationÚxrŠ   Ú	perf_sizeÚperf_threshold_timeÚwarmupÚstart_tÚ_Úend_tÚrets              r&   Úallreduce_perfzFleet.allreduce_perfo  s  € ð ˆ=˜EŸL™L¨AÒ-Ü�N‰NÐFÔGØÜ×Ñ×"Ñ"Ô$Ü�‰×Ñ×&Ñ&Ô(Ü—)‘)“+ˆÜ�yÖ!ˆAÜ×Ñ×)Ñ)¨!°5Ð)Õ9ð "ä�‰×Ñ×&Ñ&Ô(Ü—	‘	“ˆØ�w‰ )Ñ+ˆÙØÜ�‰Ø% i [°À¸uÀGÐLô	
ð  Ò#¨Ð.AÒ(AÜ�N‰NØ:¸3¸%¸sÐCVÐBWÐXõð )BÐ#r(   c                 óž  — |�|j                   dk  rt        j                  d«       y t        j                  j                  «        t        j                  j                  j                  «        t        j                  «       }t        |«      D ]7  }t        j                  j                  |t        |j                  «      |¬«       Œ9 t        j                  j                  j                  «        t        j                  «       }||z
  |z  }	t        j                  d|› d|	› d�«       |dkD  r"|	|kD  rt        j                  d|	› d	|› �«       y y y )
Nr   z&reduce_perf is invalid, group invalid!)ÚdstrŠ   z[ReduceTest] nbytes r‹   rŒ   r�   z%[Perf Warnning] Reduce Test Timeout! rŽ   )r�   r   r    rB   rc   r�   r‘   r’   r“   r”   rm   ÚreduceÚminÚranksr–   ©
rG   r—   r˜   rŠ   r™   rš   rœ   r�   rž   rŸ   s
             r&   Úreduce_perfzFleet.reduce_perfŽ  s  € Øˆ=˜EŸL™L¨AÒ-Ü�N‰NÐCÔDØÜ×Ñ×"Ñ"Ô$Ü�‰×Ñ×&Ñ&Ô(Ü—)‘)“+ˆÜ�yÖ!ˆAÜ×Ñ×%Ñ% a¬S°·±Ó-=ÀUÐ%ÕKð "ä�‰×Ñ×&Ñ&Ô(Ü—	‘	“ˆØ�w‰ )Ñ+ˆÜ�‰Ø" 9 +¨_¸S¸EÀÐIô	
ð  Ò#¨Ð.AÒ(AÜ�N‰NØ7¸°u¸CÐ@SÐ?TÐUõð )BÐ#r(   c                 óž  — |�|j                   dk  rt        j                  d«       y t        j                  j                  «        t        j                  j                  j                  «        t        j                  «       }t        |«      D ]7  }t        j                  j                  |t        |j                  «      |¬«       Œ9 t        j                  j                  j                  «        t        j                  «       }||z
  |z  }	t        j                  d|› d|	› d�«       |dkD  r"|	|kD  rt        j                  d|	› d	|› �«       y y y )
Nr   z)broadcast_perf is invalid, group invalid!)ÚsrcrŠ   z[BroadcastTest] nbytes r‹   rŒ   r�   z([Perf Warnning] Broadcast Test Timeout! rŽ   )r�   r   r    rB   rc   r�   r‘   r’   r“   r”   rm   Ú	broadcastr¤   r¥   r–   r¦   s
             r&   Úbroadcast_perfzFleet.broadcast_perf£  s  € ð ˆ=˜EŸL™L¨AÒ-Ü�N‰NÐFÔGØÜ×Ñ×"Ñ"Ô$Ü�‰×Ñ×&Ñ&Ô(Ü—)‘)“+ˆÜ�yÖ!ˆAÜ×Ñ×(Ñ(¨´°E·K±KÓ0@ÈÐ(ÕNð "ä�‰×Ñ×&Ñ&Ô(Ü—	‘	“ˆØ�w‰ )Ñ+ˆÜ�‰Ø% i [°À¸uÀGÐLô	
ð  Ò#¨Ð.AÒ(AÜ�N‰NØ:¸3¸%¸sÐCVÐBWÐXõð )BÐ#r(   c                 ó|  — |�|j                   dk  rt        j                  d«       y t        j                  j                  «        t        j                  j                  j                  «        t        j                  «       }t        |«      D ]&  }g }t        j                  j                  |||¬«       Œ( t        j                  j                  j                  «        t        j                  «       }	|	|z
  |z  }
t        j                  d|› d|
› d�«       |dkD  r"|
|kD  rt        j                  d|
› d	|› �«       y y y )
Nr   z)allgather_perf is invalid, group invalid!r‰   z[AllgatherTest] nbytes r‹   rŒ   r�   z([Perf Warnning] Allgather Test Timeout! rŽ   )r�   r   r    rB   rc   r�   r‘   r’   r“   r”   rm   Ú
all_gatherr–   )rG   r—   r˜   rŠ   r™   rš   rœ   r�   Útmprž   rŸ   s              r&   Úallgather_perfzFleet.allgather_perfº  s  € ð ˆ=˜EŸL™L¨AÒ-Ü�N‰NÐFÔGØÜ×Ñ×"Ñ"Ô$Ü�‰×Ñ×&Ñ&Ô(Ü—)‘)“+ˆÜ�yÖ!ˆAØˆCÜ×Ñ×)Ñ)¨#¨q¸Ð)Õ>ð "ô 	�‰×Ñ×&Ñ&Ô(Ü—	‘	“ˆØ�w‰ )Ñ+ˆÜ�‰Ø% i [°À¸uÀGÐLô	
ð  Ò#¨Ð.AÒ(AÜ�N‰NØ:¸3¸%¸sÐCVÐBWÐXõð )BÐ#r(   c                 óÚ  — |�|j                   dk  rt        j                  d«       y t        j                  j                  «        t        j                  j                  j                  «        |j                   }|j                  }|j                  d   |z  dk7  r*t        j                  d|j                  d   › d|› d�«       y |d   |z  |d<   t        j                  ||j                  ¬«      }t        j                  «       }	t        |«      D ]R  }
t        j                  j                  j                  ||t        j                  j                   j"                  |d¬	«       ŒT t        j                  j                  j                  «        t        j                  «       }||	z
  |z  }t        j$                  d
|› d|› d�«       |dkD  r"||kD  rt        j                  d|› d|› �«       y y y )Nr   z.reduce_scatter_perf is invalid, group invalid!r   zthe shape of input[z9] can't be divided exactly by reduce_scatter parallelism[z], test stopped!)ÚshapeÚdtypeT)ÚoprŠ   Úsync_opz[ReduceScatterTest] nbytes r‹   rŒ   r�   z,[Perf Warnning] ReduceScatter Test Timeout! rŽ   )r�   r   r    rB   rc   r�   r‘   r’   r“   r±   Úemptyr²   r”   rm   ÚstreamÚreduce_scatterÚReduceOpÚSUMr–   )rG   r—   r˜   rŠ   r™   rš   ÚparallelismÚoutput_shapeÚoutputrœ   r�   rž   rŸ   s                r&   Úreduce_scatter_perfzFleet.reduce_scatter_perfÒ  s±  € ð ˆ=˜EŸL™L¨AÒ-Ü�N‰NÐKÔLØÜ×Ñ×"Ñ"Ô$Ü�‰×Ñ×&Ñ&Ô(Ø—l‘lˆØ—w‘wˆØ�7‰7�1‰:˜Ñ# qÒ(Ü�N‰NØ% a§g¡g¨a¡j \Ð1jÐkvÐjwð  xHð  Iôð Ø& q™/¨[Ñ8ˆ�Q‰Ü—‘ L¸¿¹Ô@ˆÜ—)‘)“+ˆÜ�yÖ!ˆAÜ×Ñ×%Ñ%×4Ñ4ØØÜ×%Ñ%×.Ñ.×2Ñ2ØØð 5õ ð "ô 	�‰×Ñ×&Ñ&Ô(Ü—	‘	“ˆØ�w‰ )Ñ+ˆÜ�‰Ø)¨)¨°OÀCÀ5ÈÐPô	
ð  Ò#¨Ð.AÒ(AÜ�N‰NØ>¸s¸eÀ3ÐGZÐF[Ð\õð )BÐ#r(   é2   c           	      ó¸  — |€| j                  «       }| j                  | j                  | j                  | j                  | j
                  dœ}|j                  «       }|j                  «       }|j                  «       }d }|j                  dkD  r|}n|j                  dkD  r|}|||||dœ}	|j                  «       D ]ž  \  }
}d}d}t        j                  }d}|�|d   }|}|d   }|dk  rt        j                  d|› �«        y ||k  sŒOt        j                  |dz  g|¬«      }| j                  d	|d |dd
¬«        ||
   |||	|
   ||¬«       |dz  }||k  rŒPŒ  y )N)Ú	allreducer£   rª   Ú	allgatherr·   r   i   i   @r   zBSize for collective performance check should be positive, but got é   )r²   é
   T)r›   )r—   r˜   rŠ   r™   rš   )Úget_hybrid_communicate_groupr    r§   r«   r¯   r½   Úget_data_parallel_groupÚget_sharding_parallel_groupÚget_model_parallel_groupr�   ÚitemsrB   Úfloat32r   r    Úzeros)rG   ÚroundÚcontextÚhcgÚcollective_perf_func_mapÚdp_groupÚsharding_groupÚmp_groupÚ
data_groupÚcollective_perf_group_mapÚ	comm_typeÚsize_and_timeÚnbytesÚfinal_nbytesr²   Útime_thresholdr˜   s                    r&   Ú_collective_perf_implzFleet._collective_perf_implü  sž  € Øˆ;Ø×3Ñ3Ó5ˆCð ×,Ñ,Ø×&Ñ&Ø×,Ñ,Ø×,Ñ,Ø"×6Ñ6ñ$
Ð ð ×.Ñ.Ó0ˆØ×8Ñ8Ó:ˆØ×/Ñ/Ó1ˆØˆ
Ø�?‰?˜QÒØ!‰JØ×"Ñ" QÒ&Ø'ˆJð $Ø Ø#Ø!Ø&ñ%
Ð!ð )0¯©®Ñ$ˆI�}àˆFØ"ˆLÜ—N‘NˆEØˆNàÐ(Ø& qÑ)�à%�Ø!.¨qÑ!1�Ø˜Š{Ü—‘ØXÐY_ÐX`Ðaôñ à˜LÓ(Ü—L‘L &¨A¡+ °eÔ<�à×#Ñ# B¨¨4°¸À4Ð#ÔHà3Ð(¨Ñ3Ø#ØØ3°IÑ>Ø$Ø(6õð   1™�ð ˜LÔ(ñ% )8r(   c                 óª   — | j                   st        j                  d«       y|j                  «       D ]  \  }}|||gi}| j	                  ||¬«       Œ  y)a  
        Run performance test for given communication type
        and compare the time cost with the threshold.

        Args:
            comm_type (str): Communication type for performance test. Currently support
                            "allreduce", "broadcast", "reduce", "allgather" and "reduce_scatter".
            round (int, optional): Loop times for performance test. More loops will cost more time
                            and provide more accurate result. Defaults to 50.
            size_and_time (dict, optional): Message sizes and time thresholds for performance test.
                            each pair will invoke a performance check. Defaults to {}, which indicates
                            acting performance check from 1MB to 1GB without threshold set.

        Returns:
            None

        Examples:
            .. code-block:: python
                :name: code-init-example1

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init(is_collective=True)
                >>> # run two tests, one with 1MB (threshold 0.5s) and another with 1GB (threshold 1s)
                >>> size_and_time = {1<<20: 0.5, 1<<30: 1}
                >>> fleet.collective_perf("allreduce", round=50, size_and_time = size_and_time)
        zRfleet.collective_perf is only for collective mode, will return with no test acted.N)rË   rÌ   )r   r   r    rÈ   rÙ   )rG   rÔ   rË   rÕ   ÚsizerØ   rÌ   s          r&   Úcollective_perfzFleet.collective_perf8  s[   € ð6 ×"Ò"Ü�N‰NØdôð Ø$1×$7Ñ$7Ö$9Ñ ˆD�.Ø  4¨Ð"8Ð9ˆGØ×&Ñ&¨U¸GÐ&ÕDñ %:r(   c                 óâ  — | j                   j                  | _        | j                  d   | _        | j                  d   | _        | j                  d   | _        | j                  d   | _        | j                  d   | _        | j                  dk\  sJ d«       ‚| j                  dk\  sJ d«       ‚| j
                  dk\  sJ d	«       ‚| j                  dk\  sJ d
«       ‚t        | j                  d«      | _        t        | j                  d«      | _        t        | j
                  d«      | _        | j                  dk  r?t        j                  j                  «       }|| j                  | j                  z  z  | _        t        | j                  d«      | _        d| j                  gd| j                  gd| j                  gd| j                  gd| j
                  gdœ}| j                   j                  }|dd j                  «       t        |j                  «       «      dd j                  «       k7  rt        d«      ‚g }g }|D ],  }||   \  }}|j!                  |«       |j!                  |«       Œ. t#        j$                  ||¬«      | _        t#        j(                  | j&                  «      | _        | j                  dkD  r7| j                   j,                  }	|	d   }
|
dk(  rt/        «        yt/        |
«       yy)z"initialize the hybrid environment.Ú	dp_degreerQ   Ú	pp_degreeÚ
sep_degreeÚsharding_degreer   z)mp_degree should be greater or equal to 0z)pp_degree should be greater or equal to 0z*sep_degree should be greater or equal to 0z/sharding_degree should be greater or equal to 0r   ÚdataÚpiperi   rS   Úsep)ÚdpÚppri   Úmprä   Nz0The order of hybrid_config setting is incorrect.)Úhybrid_group_namesÚdimsÚtensor_init_seedr�   )r   Úhybrid_configsrÞ   rQ   rß   rà   rá   ÚmaxrB   rc   Úget_world_sizeÚhybrid_parallel_orderÚsortrl   ÚkeysÚAssertionErrorÚappendrf   ÚCommunicateTopologyÚ	_topologyÚHybridCommunicateGroupro   rt   r   )rG   r�   Úd_hybrid_degreeÚorderrè   ré   Úh_nameÚnameÚdegreert   rê   s              r&   rh   zFleet._init_hybrid_parallel_env\  s³  € à"×9Ñ9×HÑHˆÔØ×,Ñ,¨[Ñ9ˆŒØ×,Ñ,¨[Ñ9ˆŒØ×,Ñ,¨[Ñ9ˆŒØ×-Ñ-¨lÑ;ˆŒØ#×2Ñ2Ð3DÑEˆÔà�~‰~ Ò"ÐOÐ$OÓOÐ"Ø�~‰~ Ò"ÐOÐ$OÓOÐ"à�O‰O˜qÒ ð	8à7ó	8Ø ð × Ñ  AÒ%ð	=à<ó	=Ø%ô ˜TŸ^™^¨QÓ/ˆŒÜ˜TŸ^™^¨QÓ/ˆŒÜ˜dŸo™o¨qÓ1ˆŒà�>‰>˜AÒÜ×'Ñ'×6Ñ6Ó8ˆFØ#¨¯©¸¿¹Ñ(GÑHˆDŒNä˜TŸ^™^¨QÓ/ˆŒð ˜4Ÿ>™>Ð*Ø˜4Ÿ>™>Ð*Ø# T×%9Ñ%9Ð:Ø˜DŸN™NÐ+Ø˜4Ÿ?™?Ð+ñ
ˆð ×+Ñ+×AÑAˆØ‘ˆ8�=‰=‹?œd ?×#7Ñ#7Ó#9Ó:¹1Ð=×BÑBÓDÒDÜ ØBóð ð  ÐØˆÛˆFØ*¨6Ñ2‰LˆD�&Ø×%Ñ% dÔ+Ø�K‰K˜Õð ô
 ×/Ñ/Ø1¸ô
ˆŒô ×-Ñ-¨d¯n©nÓ=ˆŒ	à�>‰>˜AÒà×+Ñ+×CÑCð $ð  7Ð7IÑJÐØ 2Ò%Ü*Õ,ä*Ð+;Õ<ð r(   c                 ó6   — | j                   €J ‚| j                   S ©N)ro   rF   s    r&   rÄ   z"Fleet.get_hybrid_communicate_group�  s   € Ø�y‰yÐ$Ð$Ð$Ø�y‰yÐr(   c                 ó6   — | j                   €J ‚| j                   S rü   )rô   rF   s    r&   Úget_hybrid_parallel_topologyz"Fleet.get_hybrid_parallel_topology¡  s   € Ø�~‰~Ð)Ð)Ð)Ø�~‰~Ðr(   c                 ó6   — | j                   j                  «       S )ag  
        Check whether the node is the first instance of worker.

        Returns:
            bool: True if this is the first node of worker, False if not.

        Examples:
            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.is_first_worker()

        )r8   Ú_is_first_workerrF   s    r&   Úis_first_workerzFleet.is_first_worker¥  s   € ð ×Ñ×0Ñ0Ó2Ð2r(   c                 ó6   — | j                   j                  «       S )a  
        Get current worker index.

        Returns:
            int: node id

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.worker_index()

        )r8   Ú_worker_indexrF   s    r&   rj   zFleet.worker_index¶  ó   € ð  ×Ñ×-Ñ-Ó/Ð/r(   c                 ó6   — | j                   j                  «       S )a"  
        Get current total worker number.

        Returns:
            int: worker numbers

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.worker_num()

        )r8   Ú_worker_numrF   s    r&   rk   zFleet.worker_numÈ  s   € ð  ×Ñ×+Ñ+Ó-Ð-r(   c                 ó6   — | j                   j                  «       S rü   )r8   Ú_get_node_numrF   s    r&   Únode_numzFleet.node_numÚ  s   € Ø×Ñ×-Ñ-Ó/Ð/r(   c                 ó6   — | j                   j                  «       S rü   )r8   Ú_get_local_rankrF   s    r&   Ú
local_rankzFleet.local_rankÝ  ó   € Ø×Ñ×/Ñ/Ó1Ð1r(   c                 ó6   — | j                   j                  «       S rü   )r8   Ú_get_local_device_idsrF   s    r&   Úlocal_device_idszFleet.local_device_idsà  ó   € Ø×Ñ×5Ñ5Ó7Ð7r(   c                 ó6   — | j                   j                  «       S rü   )r8   Ú_get_world_device_idsrF   s    r&   Úworld_device_idszFleet.world_device_idsã  r  r(   c                 ó6   — | j                   j                  «       S )ae  
        Check whether the node is an instance of worker.

        Returns:
            bool: True if this is a node of worker,
                  False if not.

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.is_worker()

        )r8   Ú
_is_workerrF   s    r&   Ú	is_workerzFleet.is_workeræ  ó   € ð" ×Ñ×*Ñ*Ó,Ð,r(   c                 ó6   — | j                   j                  «       S rü   )r8   Ú_is_coordinatorrF   s    r&   Úis_coordinatorzFleet.is_coordinatorù  r  r(   c                 óŒ   — |r)dj                  | j                  j                  «       «      S | j                  j                  «       S )a]  
        Get current worker endpoints, such as ["127.0.0.1:1001", "127.0.0.1:1002"].

        Returns:
            list/string: server endpoints

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.worker_endpoints()

        Ú,)Újoinr8   Ú_get_trainer_endpoints©rG   Ú	to_strings     r&   Úworker_endpointszFleet.worker_endpointsü  s:   € ñ  Ø—8‘8˜D×,Ñ,×CÑCÓEÓFÐFà×#Ñ#×:Ñ:Ó<Ð<r(   c                 óH   — t        | j                  j                  «       «      S )a   
        Get current total worker number.

        Returns:
            int: server number

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.server_num()
        )Úlenr8   Ú_get_pserver_endpointsrF   s    r&   Ú
server_numzFleet.server_num  s   € ô �4×#Ñ#×:Ñ:Ó<Ó=Ð=r(   c                 ó6   — | j                   j                  «       S )a  
        Get current server index.

        Returns:
            int: node id

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.server_index()

        )r8   Ú_server_indexrF   s    r&   Úserver_indexzFleet.server_index"  r  r(   c                 óŒ   — |r)dj                  | j                  j                  «       «      S | j                  j                  «       S )a]  
        Get current server endpoints, such as ["127.0.0.1:1001", "127.0.0.1:1002"].

        Returns:
            list/string: server endpoints

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.server_endpoints()

        r  )r  r8   r%  r   s     r&   Úserver_endpointszFleet.server_endpoints4  s:   € ñ" Ø—8‘8˜D×,Ñ,×CÑCÓEÓFÐFà×#Ñ#×:Ñ:Ó<Ð<r(   c                 ó6   — | j                   j                  «       S )ae  
        Check whether the node is an instance of server.

        Returns:
            bool: True if this is a node of server,
                  False if not.

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.is_server()

        )r8   Ú
_is_serverrF   s    r&   Ú	is_serverzFleet.is_serverJ  r  r(   c                 ó:   — | j                   j                  d«       y)a	  
        barrier all workers

        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> fleet.barrier_worker()
        ÚworkerN)r8   Ú_barrierrF   s    r&   Úbarrier_workerzFleet.barrier_worker]  s   € ð 	×Ñ×!Ñ! (Õ+r(   c                 ó<   — | j                   j                  ||d«      S )ac  
        all reduce input between all workers, mode can be sum, mean or max, default is sum

        Returns:
            list/int: all reduce result

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> res = fleet.all_reduce(5)

        r0  )r8   Ú_all_reduce)rG   ÚinputÚmodes      r&   r•   zFleet.all_reducen  s   € ð  ×Ñ×+Ñ+¨E°4¸ÓBÐBr(   c                 ó:   — | j                   j                  |«       y)a†  
        initialize `Communicator` for parameter server training.


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.init_worker()

        N)r+   Ú_init_worker©rG   Úscopess     r&   Úinit_workerzFleet.init_worker€  s   € ð. 	×Ñ×)Ñ)¨&Õ1r(   c                 ó:   — | j                   j                  |«       y)z-
        initialize coordinator node
        N)r+   Ú_init_coordinatorr9  s     r&   Úinit_coordinatorzFleet.init_coordinator™  s   € ð 	×Ñ×.Ñ.¨vÕ6r(   c                 ó8   — | j                   j                  «        y rü   )r+   Ú_make_fl_strategyrF   s    r&   Úmake_fl_strategyzFleet.make_fl_strategy¡  s   € Ø×Ñ×.Ñ.Õ0r(   c                 ó.   — | j                   j                  S )z/
        get worker(training node) ptr
        )r+   Ú_workerrF   s    r&   Úget_fl_clientzFleet.get_fl_client¤  s   € ð ×#Ñ#×+Ñ+Ð+r(   c                 ó<   —  | j                   j                  |i |¤Ž y)aÛ  
        init_server executor to initialize startup program,
        if the `args` is not empty, it will run load_persistables for increment training.


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.init_server()

        N)r+   Ú_init_server)rG   r.   r/   s      r&   Úinit_serverzFleet.init_server¬  s   € ð0 	*ˆ×Ñ×)Ñ)¨4Ð:°6Ó:r(   c                 ó<   — | j                   j                  ||«       y)au  
        load fleet model from path


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.load_model("path", mode=0)

        N)r+   Ú_load_persistables©rG   Úpathr6  s      r&   Ú
load_modelzFleet.load_modelÆ  s   € ð. 	×Ñ×/Ñ/°°dÕ;r(   c                 ó>   — | j                   j                  |||«       y)a€  
        load fleet one table from path


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.load_one_table(0, "path", mode=0)

        N)r+   Ú_load_one_table©rG   Útable_idrK  r6  s       r&   Úload_one_tablezFleet.load_one_tableß  ó   € ð. 	×Ñ×,Ñ,¨X°t¸TÕBr(   c                 ó<   — | j                   j                  ||«       y)a‰  
        load fleet inference model from path


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.load_inference_model("path", mode=1)

        N)r+   Ú_load_inference_modelrJ  s      r&   Úload_inference_modelzFleet.load_inference_modelø  s   € ð. 	×Ñ×2Ñ2°4¸Õ>r(   c                 ó8   — | j                   j                  «        y)a²  
        run server will run pserver main program with executor.

        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> if fleet.is_server():
                ...     fleet.init_server()

        N)r+   Ú_run_serverrF   s    r&   Ú
run_serverzFleet.run_server  s   € ð. 	×Ñ×(Ñ(Õ*r(   c                 ó8   — | j                   j                  «        y)a—  
        stop `Communicator` and give training complete notice to parameter server.

        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.init_server()

        N)r+   Ú_stop_workerrF   s    r&   Ústop_workerzFleet.stop_worker*  s   € ð, 	×Ñ×)Ñ)Õ+r(   c           	      ó˜  — d}|s|sd}t        j                  «       }t         j                  j                  |«      }|�rTg }g }	|D ]n  }
t	        |
t
        «      r|j                  |
«       Œ%t	        |
t         j                  j                  «      r|j                  |
j                  «       Œet        d«      ‚ |D ]n  }
t	        |
t
        «      r|	j                  |
«       Œ%t	        |
t         j                  j                  «      r|	j                  |
j                  «       Œet        d«      ‚ |	D �cg c]=  }t         j                  j                  «       j                  «       j                  |«      ‘Œ? }}| j                  j                  ||||d dd«       y d}d|v rt        |d   «      }| j                  j!                  ||d |¬«       y c c}w )NTFzfeed must be [str|Variable]r   r6  )r!   r6  )rB   ÚCPUPlaceÚstaticÚExecutorrW   ra   rò   ÚVariablerù   r,   Údefault_main_programÚglobal_blockÚvarr+   Ú_save_inference_modelrs   Ú_save_persistables)rG   ÚdirnameÚfeedÚfetchÚconfigsÚ	inferenceÚplaceÚexecutorÚfeeded_var_namesÚfetch_var_namesrc  rù   Ú
fetch_varsÚincrement_modes                 r&   Úsavez
Fleet.saveB  s˜  € ð ˆ	á™EØˆIä—‘Ó!ˆÜ—=‘=×)Ñ)¨%Ó0ˆâØ!ÐØ ˆOã�Ü˜c¤3Ô'Ø$×+Ñ+¨CÕ0Ü ¤V§]¡]×%;Ñ%;Ô<Ø$×+Ñ+¨C¯H©HÕ5ä$Ð%BÓCÐCð ó �Ü˜c¤3Ô'Ø#×*Ñ*¨3Õ/Ü ¤V§]¡]×%;Ñ%;Ô<Ø#×*Ñ*¨3¯8©8Õ4ä$Ð%BÓCÐCð ñ ,óá+�Dô —‘×2Ñ2Ó4×AÑAÓC×GÑGÈÕMØ+ð ð ð
 × Ñ ×6Ñ6Ø˜'Ð#3°ZÀÀtÈQõð ˆNØ˜Ñ Ü!$ W¨V¡_Ó!5�Ø× Ñ ×3Ñ3Ø˜'°¸>ð 4õ ùòs   Ä-AGc           	      óF   — | j                   j                  |||||||«       y)ap  
        save inference model for inference.

        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.init_server()

        N)r+   rd  )rG   rl  rf  rm  Útarget_varsr!   Úexport_for_deploymentr6  s           r&   Úsave_inference_modelzFleet.save_inference_modelq  s.   € ð@ 	×Ñ×2Ñ2ØØØØØØ!Øõ	
r(   c                 ó@   — | j                   j                  ||||«       y)az  

        saves all persistable tensors from :code:`main_program` to
        the folder :code:`dirname`. You can refer to

        The :code:`dirname` is used to specify the folder where persistable tensors
        are going to be saved. If you would like to save tensors in separate
        files, set :code:`filename` None.

        Args:
            executor(Executor): The executor to run for saving persistable tensors.
                                You can refer to :ref:`api_guide_executor_en` for
                                more details.

            dirname(str, optional): The saving directory path.
                                When you need to save the parameter to the memory, set it to None.
            main_program(Program, optional): The program whose persistbale tensors will
                                             be saved. Default: None.


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle
                >>> paddle.enable_static()
                >>> import paddle.distributed.fleet as fleet

                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> exe = paddle.static.Executor(paddle.CPUPlace())
                >>> fleet.save_persistables(exe, "dirname", paddle.static.default_main_program())

        N)r+   re  )rG   rl  rf  r!   r6  s        r&   Úsave_persistableszFleet.save_persistables›  s"   € ðV 	×Ñ×/Ñ/Ø�g˜|¨Tõ	
r(   c                 ó<   —  | j                   j                  |fi |¤ŽS rü   )r+   Ú_save_cache_model)rG   rf  ri  s      r&   Úsave_cache_modelzFleet.save_cache_modelÊ  s"   € ð 6ˆt×#Ñ#×5Ñ5°gÑIÀÑIÐIr(   c                 ó6   — | j                   j                  «       S rü   )r+   Ú_check_save_pre_patch_donerF   s    r&   Úcheck_save_pre_patch_donezFleet.check_save_pre_patch_doneÏ  s   € ð ×#Ñ#×>Ñ>Ó@Ð@r(   c                 ó<   — | j                   j                  |||«      S rü   )r+   Ú_save_cache_table)rG   rP  Úpass_idÚmem_cache_key_thresholds       r&   Úsave_cache_tablezFleet.save_cache_tableÔ  s%   € ð
 ×#Ñ#×5Ñ5Ø�gÐ6ó
ð 	
r(   c                 ó>   — | j                   j                  |||«       y)a€  
        save fleet one table from path


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.save_one_table(0, "path", mode=0)

        N)r+   Ú_save_one_tablerO  s       r&   Úsave_one_tablezFleet.save_one_tableÝ  rR  r(   c                 óB   — | j                   j                  |||||«       y)aX  
        save fleet one table from path


        Returns:
            None

        Examples:

            .. code-block:: python

                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init()
                >>> import paddle
                >>> place = paddle.CPUPlace()
                >>> exe =  paddle.static.Executor(place)

                >>> # build net
                >>> # fleet.distributed_optimizer(...)

                >>> fleet.save_dense_params(exe, "path", scope=paddle.static.global_scope(), program=paddle.static.default_main_program())

        N)r+   Ú_save_dense_params)rG   rl  rf  ÚscopeÚprogramÚ	var_namess         r&   Úsave_dense_paramszFleet.save_dense_paramsö  s#   € ð8 	×Ñ×/Ñ/Ø�g˜u g¨yõ	
r(   c                 ó:   — | j                   j                  |«       y rü   )r+   Ú_shrink)rG   Ú	thresholds     r&   ÚshrinkzFleet.shrink  s   € ð 	×Ñ×$Ñ$ YÕ/r(   c                 óœ   — || _         |�;| j                  rt        j                  d«       t	        j
                  |«      | _        i | _        | S )a1  
        Optimizer for distributed training.

        For the distributed training, this method would rebuild a new instance of DistributedOptimizer.
        Which has basic Optimizer function and special features for distributed training.

        Args:
            optimizer(Optimizer): The executor to run for init server.
            strategy(DistributedStrategy): Extra properties for distributed optimizer.
                It is recommended to use DistributedStrategy in fleet.init(). The strategy
                here is for compatibility. If the strategy in fleet.distributed_optimizer()
                is not None, then it will overwrite the DistributedStrategy in fleet.init(),
                which will take effect in distributed training.

        Returns:
            Fleet: instance of fleet.

        Examples:

            .. code-block:: python

                >>> import paddle
                >>> import paddle.distributed.fleet as fleet
                >>> fleet.init(is_collective=True)
                >>> linear = paddle.nn.Linear(10, 10)
                >>> strategy = fleet.DistributedStrategy()
                >>> optimizer = paddle.optimizer.SGD(learning_rate=0.001, parameters=linear.parameters())
                >>> optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)

        a  It is recommended to use DistributedStrategy in fleet.init(). The strategy here is only for compatibility. If the strategy in fleet.distributed_optimizer() is not None, then it will overwrite the DistributedStrategy in fleet.init(), which will take effect in distributed training.)rE   r   r   r    rU   rV   r   rA   )rG   rC   rv   s      r&   Údistributed_optimizerzFleet.distributed_optimizer  sL   € ð> '0ˆÔ#àÐØ×"Ò"Ü—‘ðFôô +/¯-©-¸Ó*AˆDÔ'àˆŒàˆr(   c                 óÂ   — d }| j                   j                  «       D ]  }t        |d«      sŒ|} n |€"t        | j                  d«      r| j                  }|€J d«       ‚|S )NÚamp_initzSamp_init can only be used when the amp(auto mixed precision) strategy is turned on.©r?   Ú_get_applied_meta_optimizerÚhasattrrE   )rG   Úamp_optimizerrC   s      r&   Ú_get_amp_optimizerzFleet._get_amp_optimizerK  sv   € àˆØ×/Ñ/×KÑKÖMˆIÜ�y *Õ-Ø )�Ùð Nð
 Ð Ü�t×2Ñ2°JÔ?Ø $× ;Ñ ;�ð Ð%ð	aà`ó	aØ%àÐr(   c                 óB   — | j                  «       }|j                  «       S )z)Return the real-time loss scaling factor.)r˜  Úget_loss_scaling)rG   r—  s     r&   rš  zFleet.get_loss_scaling\  s   € à×/Ñ/Ó1ˆØ×-Ñ-Ó/Ð/r(   c                 óJ   — | j                  «       }|j                  ||||«      S )aJ  
        Init the amp training, such as cast fp32 parameters to fp16 type.

        Args:
            place(CUDAPlace): place is used to initialize
                fp16 parameters with fp32 values.
            scope(Scope): The scope is used to find fp32 parameters.
            test_program(Program): The program is used for testing.
            use_fp16_test(bool): Whether to use fp16 testing.

        Examples:
            .. code-block:: python

                >>> import paddle
                >>> import paddle.nn.functional as F
                >>> paddle.enable_static()

                >>> def run_example_code():
                ...     place = paddle.CUDAPlace(0)
                ...     exe = paddle.static.Executor(place)
                ...     data = paddle.static.data(name='X', shape=[None, 1, 28, 28], dtype='float32')
                ...     conv2d = paddle.static.nn.conv2d(input=data, num_filters=6, filter_size=3)
                ...     # 1) Use fp16_guard to control the range of fp16 kernels used.
                ...     with paddle.static.amp.fp16_guard():
                ...         bn = paddle.static.nn.batch_norm(input=conv2d, act="relu")
                ...         pool = F.max_pool2d(bn, kernel_size=2, stride=2)
                ...         hidden = paddle.static.nn.fc(pool, size=10)
                ...         loss = paddle.mean(hidden)
                ...     # 2) Create the optimizer and set `multi_precision` to True.
                ...     # Setting `multi_precision` to True can avoid the poor accuracy
                ...     # or the slow convergence in a way.
                ...     optimizer = paddle.optimizer.Momentum(learning_rate=0.01, multi_precision=True)
                ...     # 3) These ops in `custom_black_list` will keep in the float32 computation type.
                ...     amp_list = paddle.static.amp.CustomOpLists(
                ...         custom_black_list=['pool2d'])
                ...     # 4) The entry of Paddle AMP.
                ...     # Enable pure fp16 training by setting `use_pure_fp16` to True.
                ...     optimizer = paddle.static.amp.decorate(
                ...         optimizer,
                ...         amp_list,
                ...         init_loss_scaling=128.0,
                ...         use_dynamic_loss_scaling=True,
                ...         use_pure_fp16=True)
                ...     # If you don't use the default_startup_program(), you sholud pass
                ...     # your defined `startup_program` into `minimize`.
                ...     optimizer.minimize(loss)
                ...     exe.run(paddle.static.default_startup_program())
                ...     # 5) Use `amp_init` after FP32 parameters initialization(such as `exe.run(startup_program)`).
                ...     # If you want to perform the testing process, you should pass `test_program` into `amp_init`.
                ...     optimizer.amp_init(place, scope=paddle.static.global_scope())

                >>> if paddle.is_compiled_with_cuda() and len(paddle.static.cuda_places()) > 0:
                ...     run_example_code()
        )r˜  r“  )rG   rk  rˆ  Útest_programÚuse_fp16_testr—  s         r&   r“  zFleet.amp_inita  s*   € ðr ×/Ñ/Ó1ˆØ×%Ñ% e¨U°LÀ-ÓPÐPr(   c                 óÂ   — d }| j                   j                  «       D ]  }t        |d«      sŒ|} n |€"t        | j                  d«      r| j                  }|€J d«       ‚|S )NÚqat_initzZqat_init can only be used when the qat(quantization aware training) strategy is turned on.r”  )rG   Úqat_optimizerrC   s      r&   Ú_get_qat_optimizerzFleet._get_qat_optimizer�  sv   € àˆØ×/Ñ/×KÑKÖMˆIÜ�y *Õ-Ø )�Ùð Nð
 Ð Ü�t×2Ñ2°JÔ?Ø $× ;Ñ ;�ð Ð%ð	hàgó	hØ%àÐr(   c                 óJ   — | j                  «       }|j                  |||¬«      S )aQ  
        Init the qat training, such as insert qdq ops and scale variables.

        Args:
            place(CUDAPlace): place is used to initialize
                scale parameters.
            scope(Scope): The scope is used to find parameters and variables.
            test_program(Program): The program is used for testing.
        )rˆ  rœ  )r¡  rŸ  )rG   rk  rˆ  rœ  r   s        r&   rŸ  zFleet.qat_init®  s1   € ð ×/Ñ/Ó1ˆØ×%Ñ%Ø˜¨\ð &ó 
ð 	
r(   c                 óV   — d| j                   vrt        d«       i S | j                   d   S )NÚvalid_strategyzNWARNING: You may need to call minimize function before this function is called©rA   ÚprintrF   s    r&   Ú_final_strategyzFleet._final_strategy½  s0   € Ø 4§=¡=Ñ0ÜØ`ôð ˆIà—=‘=Ð!1Ñ2Ð2r(   c                 óV   — d| j                   vrt        d«       g S | j                   d   S )NÚapplied_meta_listzTWARNING: You may need to call minimize function before _get_applied_meta_list calledr¥  rF   s    r&   Ú_get_applied_meta_listzFleet._get_applied_meta_listÆ  s0   € Ø d§m¡mÑ3ÜØfôð ˆIà—=‘=Ð!4Ñ5Ð5r(   c                 óV   — d| j                   vrt        d«       g S | j                   d   S )NÚapplied_graph_listzUWARNING: You may need to call minimize function before _get_applied_graph_list calledr¥  rF   s    r&   Ú_get_applied_graph_listzFleet._get_applied_graph_listÏ  s0   € Ø t§}¡}Ñ4ÜØgôð ˆIà—=‘=Ð!5Ñ6Ð6r(   c                 óè   — t        |t        «      s| j                  ||||«      S t        «       s&| j                  j                  «       s| j                  rt        d«      ‚| j                  ||||«      S )aË
  
        Add distributed operations to minimize ``loss`` by updating ``parameter_list``.

        Args:
            loss (Tensor): A ``Tensor`` containing the value to minimize.
            startup_program (Program, optional): :ref:`api_paddle_static_Program` for
                initializing parameters in ``parameter_list``. The default value
                is None, at this time :ref:`api_paddle_static_default_startup_program` will be used.
            parameter_list (Iterable, optional): Iterable of ``Tensor`` or ``Tensor.name`` to update
                to minimize ``loss``. The default value is None, at this time all parameters
                will be updated.
            no_grad_set (set, optional): Set of ``Tensor``  or ``Tensor.name`` that don't need
                to be updated. The default value is None.

        Returns:
            tuple: tuple (optimize_ops, params_grads), A list of operators appended
            by minimize and a list of (param, grad) tensor pairs, param is
            ``Parameter``, grad is the gradient value corresponding to the parameter.
            The returned tuple can be passed to ``fetch_list`` in ``Executor.run()`` to
            indicate program pruning. If so, the program will be pruned by ``feed`` and
            ``fetch_list`` before run, see details in ``Executor``.

        Examples:

            .. code-block:: python

                >>> import paddle
                >>> paddle.enable_static()
                >>> import paddle.distributed.fleet as fleet
                >>> import paddle.nn.functional as F

                >>> hid_dim = 10
                >>> label_dim = 2
                >>> input_x = paddle.static.data(name='x', shape=[None, 13], dtype='float32')
                >>> input_y = paddle.static.data(name='y', shape=[None, 1], dtype='int64')
                >>> fc_1 = paddle.static.nn.fc(x=input_x, size=hid_dim, activation='tanh')
                >>> fc_2 = paddle.static.nn.fc(x=fc_1, size=hid_dim, activation='tanh')
                >>> prediction = paddle.static.nn.fc(x=[fc_2], size=label_dim, activation='softmax')
                >>> cost = F.cross_entropy(input=prediction, label=input_y)
                >>> avg_cost = paddle.mean(x=cost)

                >>> fleet.init(is_collective=True)
                >>> strategy = fleet.DistributedStrategy()
                >>> linear = paddle.nn.Linear(10, 10)
                >>> optimizer = paddle.optimizer.SGD(learning_rate=0.001, parameters=linear.parameters())
                >>> optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy)
                >>> optimizer.minimize(avg_cost)

                >>> # for more examples, please reference https://github.com/PaddlePaddle/PaddleFleetX

        z loss can be list only in PS mode)	rW   rl   Ú_minimize_implr   r8   r9   r   r,   Ú_minimize_losses_impl)rG   Úlossr   Úparameter_listÚno_grad_sets        r&   ÚminimizezFleet.minimizeØ  sv   € ôl ˜$¤Ô%Ø×&Ñ&Ø�o ~°{óð ô
  Ô!Ø×#Ñ#×7Ñ7Ô9Ø×&Ò&ä Ð!CÓDÐDØ×-Ñ-Ø�o ~°{óð r(   c                 óD  — i }t        j                  | j                  «      |d<   t        «       r$| j                  }|| _        |j                  |«      S |j                  j                  | _	        t        | j                  d«      sÑi | j                  _        | j                  j                  d   | j                  j                  d<   | j                  j                  d   | j                  j                  d<   | j                  j                  d   | j                  j                  d<   | j                  j                  d   | j                  j                  d<   | j                  |d<   | j                  g|d<   ||d	<   |€Rt        j                  j                  «       j!                  d
¬«      | _        t        j                  j                  «       }n|j!                  d
¬«      | _        ||d<   |g|d<   | j$                  |d<   | j                  j&                  s| j                  j(                  r-ddlm}  || «      }|j/                  ||||«      \  }	}
}}|	|
||fS t        j                  | j                  «      |d<   t        j                  | j                  «      }g }g }g }g }| j0                  r3t3        | j                  j4                  «      dkD  r|j7                  d«       t9        «       j;                  | j                  |«      }|j=                  «       r|D ]  }|j?                  ||«       Œ g }g }|D ]Ÿ  }|jA                  || j$                  | j                  |«       |jC                  «       r"|jE                  «       s|j7                  |«       Œ]|jC                  «       r"|jE                  «       r|j7                  |«       Œ�|j7                  |«       Œ¡ | j0                  r›t3        | j                  j4                  «      dkD  ryd|d<   ddl#m$}  || j                  «      }|jA                  || j$                  | j                  |«       |jK                  «        |j7                  |«       |j7                  |«       d }tM        d|«       | jN                  jQ                  || j$                  | j                  |||«      \  }}tM        d|«       tM        d|«       | jN                  jS                  ||«      }t        j                  |«      |d<   tU        jV                  dtY        |d   «      z   «       tU        jV                  dtY        |d   «      z   «       | jN                  j[                  «       }| jN                  j]                  «       }||d<   ||d<   || _        || _/        | j^                  ja                  «        g }	g }
| j$                  jc                  «       r�| j0                  sƒ| jd                  €tg        «       ji                  |«      | _2        tk        jl                  | j                  «      }||j                  j                  _7        | j                  j                  ||||¬«      S |�retU        jV                  d tY        tq        |j                  j                  «      «      z   «       |j                  ||||¬«      \  }	}
tU        jV                  d!tY        tq        |j                  j                  «      «      z   «       t        j                  js                  «       }tU        jV                  d"tY        tq        |«      «      z   «       tq        |«      tq        |j                  j                  «      k7  r3t        jt                  jw                  |j                  j                  «       tU        jV                  d#tY        tq        |«      «      z   «       n"| j                  j                  ||||¬«      \  }	}
|	|d$<   |
|d%<   |ratU        jV                  d&tY        tq        |j                  j                  «      «      z   «       |j                  ||||¬«      \  }	}
|	|d'<   |
|d(<   nA|j                  j                  jx                  €!t{        |j                  j                  || «       | j$                  j|                  s�t        j                  js                  «       }|j~                  €i n|j~                  }| j�                  «       |d)<   | jƒ                  «       |d*<   | j                  j„                  j‡                  «       D ]  \  }}|s||vsŒ|||<   Œ ||_?        | jd                  €tg        «       ji                  |«      | _2        dd+lDmE}  | jŒ                  j�                  |d   «       |	|
fS ),NÚuser_defined_strategyÚdistributed_info_rÞ   rQ   rß   rá   Úorigin_main_programÚorigin_main_programsr±  F©Úfor_testÚorigin_startup_programÚorigin_startup_programsru   é   )ÚAutoParallelizerr   ÚShardingOptimizerTÚuse_fleet_psr   ©ÚParameterServerOptimizerzvalid_optimizer_list=zmeta_optimizer=zgraph_optimizer=r¤  zvalid_strategy: zuser_defined_strategy: r©  r¬  ©r³  zbefore minimize program id: zafter minimize program id: zdefault program id: z!default program id after switch: Úprogram_optimize_opsÚprogram_params_gradsz"before graph minimize program id: Úgraph_optimize_opsÚgraph_optimize_gradsÚmpi_sizeÚmpi_rankrL   )HrU   rV   r   r   rE   rA   r´  Úblockr‰  r¸  r–  r·  rr   rB   r^  Údefault_startup_programÚcloner¼  r8   Ú	semi_autoÚauto_searchÚ!auto_parallel.static.parallelizerr¿  Úparallelizer   r$  Úsparse_table_configsrò   r   Ú_get_valid_meta_optimizersÚ_is_strict_autoÚ_enable_strategyÚ_set_basic_infoÚ
_can_applyÚ_is_graph_outÚmeta_optimizersrÃ  Úclearr¦  r?   Úgenerate_optimizerÚ_get_valid_strategyr   Údebugra   rª  r­  r¤  Ú_enable_envr9   r+   r   Ú_create_runtimer   ÚCompiledProgramÚ_graphÚidra  Ú	frameworkÚswitch_main_programÚ_pass_appliedr'   Ú_is_heter_parameter_server_modeÚ
_fleet_optrk   rj   Útrainer_desc_configsrÈ   rT   rM   r\   Ú_set_strategy)!rG   r±  r   r²  r³  rÌ   Ú
target_optr¿  Úauto_parallelizerÚoptimize_opsÚparams_gradsÚdist_startup_progÚdist_main_progÚcopy_user_defined_strategyÚcan_not_apply_optimizer_listÚvalid_optimizer_listÚvalid_graph_optimizer_listÚ
skip_namesÚdistributed_optimizer_listÚoptrÃ  Úmeta_optimizerÚgraph_optimizerr¤  r©  r¬  Úcompiled_programÚdefault_programr‰  Úopt_infoÚkÚvrM   s!                                    r&   r¯  zFleet._minimize_impl  s6	  € ð ˆÜ+/¯=©=Ø×'Ñ'ó,
ˆÐ'Ñ(ô Ôà×4Ñ4ˆJØ#ˆDŒMØ×&Ñ& tÓ,Ð,ð (,§z¡z×'9Ñ'9ˆDÔ$ä˜4×3Ñ3Ð5HÔIØ=?�×(Ñ(Ô:ð ×/Ñ/×@Ñ@ÀÑMð ×(Ñ(×:Ñ:Øñð
 ×/Ñ/×@Ñ@ÀÑMð ×(Ñ(×:Ñ:Øñð
 ×/Ñ/×@Ñ@ÀÑMð ×(Ñ(×:Ñ:Øñð
 ×/Ñ/×@Ñ@Ø%ñð ×(Ñ(×:Ñ:Ø%ñð .2×-EÑ-EˆGÐ)Ñ*Ø/3×/GÑ/GÐ.HˆGÐ*Ñ+Ø"ˆG�F‰OØÐ&ä—M‘M×9Ñ9Ó;×AÑAØ!&ð Bó ð Ô+ô
 #)§-¡-×"GÑ"GÓ"I‘à.=×.CÑ.CØ"ð /Dó /�Ô+ð 1@ˆGÐ,Ñ-Ø2AÐ1BˆGÐ-Ñ.Ø$(×$4Ñ$4ˆG�LÑ!ð ×+Ñ+×5Ò5Ø×.Ñ.×:Ò:åPá$4°TÓ$:Ð!ð &×1Ñ1Ø˜/¨>¸;óñØ Ø Ø%Ø"ð !Ø Ø%Ø"ð	ð ô 04¯}©}Ø×+Ñ+ó0ˆGÐ+Ñ,ô *.¯©Ø×+Ñ+ó*Ð&ð ,.Ð(à#%Ð Ø)+Ð&ØˆJà×#Ò#Ü˜×3Ñ3×HÑHÓIÈAÒMà×!Ñ!Ð"5Ô6ô %Ó&×AÑAØ×/Ñ/°óð 'ð *×9Ñ9Ô;ã5�CØ×(Ñ(Ð)CÀWÕMð 6ð $&Ð Ø)+Ð&ã1�Ø×#Ñ#ØØ×$Ñ$Ø×/Ñ/Ø.ô	ð —>‘>Ô#¨C×,=Ñ,=Ô,?Ø(×/Ñ/°Õ4Ø—^‘^Ô%¨#×*;Ñ*;Ô*=Ø.×5Ñ5°cÕ:à0×7Ñ7¸Õ<ð 2ð ×#Ò#Ü˜×3Ñ3×HÑHÓIÈAÒMà*.�˜Ñ'åEá!9Ø×/Ñ/ó"�ð ×.Ñ.ØØ×$Ñ$Ø×/Ñ/Ø.ô	ð %×*Ñ*Ô,Ø$×+Ñ+¨NÔ;Ø,×3Ñ3°NÔCð #'�ô Ð)Ð+?Ô@ð
 ×&Ñ&×9Ñ9ØØ× Ñ Ø×+Ñ+Ø*Ø$Ø*óñØØô Ð# ^Ô4ÜÐ$ oÔ6à!×3Ñ3×GÑGØ*Ð,HóˆNô )-¯©°nÓ(EˆGÐ$Ñ%Ü�L‰LÐ+¬c°'Ð:JÑ2KÓ.LÑLÔMÜ�L‰LØ)Ü�gÐ5Ñ6Ó7ñ8ôð
 !%× 6Ñ 6× MÑ MÓ OÐà×&Ñ&×>Ñ>Ó@ð ð ,=ˆGÐ'Ñ(Ø,>ˆGÐ(Ñ)à#ˆDŒMà"0ˆDÔØ×Ñ×+Ñ+Ô-àˆLØˆLð × Ñ ×4Ñ4Ô6Ø×+Ò+à×'Ñ'Ð/Ü+9Ó+;×+KÑ+KØó,�DÔ(ô $,×#;Ñ#;Ø×,Ñ,ó$Ð ð -=�—
‘
×"Ñ"Ô)Ø×2Ñ2×;Ñ;ØØ#Ø"Ø +ð	 <ó ð ò Ü—‘Ø2´S¼¸D¿J¹J×<NÑ<NÓ9OÓ5PÑPôð .<×-DÑ-DØØ#Ø"Ø +ð	 .Eó .Ñ*�˜lô —‘Ø1´C¼¸4¿:¹:×;MÑ;MÓ8NÓ4OÑOôô #)§-¡-×"DÑ"DÓ"F�Ü—‘Ð3´c¼"¸_Ó:MÓ6NÑNÔOä�oÓ&¬"¨T¯Z©Z×-?Ñ-?Ó*@Ò@Ü×$Ñ$×8Ñ8¸¿¹×9KÑ9KÔLÜ—‘Ø7Üœ"˜_Ó-Ó.ñ/õð ×/Ñ/×8Ñ8ØØ#Ø"Ø +ð	 9ó ñØ Ø ð /;ˆGÐ*Ñ+Ø.:ˆGÐ*Ñ+áÜ—‘Ø8Üœ"˜TŸZ™Z×/Ñ/Ó0Ó1ñ2ôð .=×-EÑ-EØØ#Ø"Ø +ð	 .Fó .Ñ*�˜lð 1=�Ð,Ñ-Ø2>�Ð.Ò/Ø—‘×#Ñ#×1Ñ1Ð9Ü §
¡
× 2Ñ 2°OÀTÔJà×#Ñ#×CÒCÜ Ÿ-™-×<Ñ<Ó>�à!×,Ñ,Ð4‘B¸'×:LÑ:Lð ð (,§¡Ó'8�˜Ñ$Ø'+×'8Ñ'8Ó':�˜Ñ$ð ×0Ñ0×EÑE×KÑKÖMñØØá˜A XÒ-Ø&'˜ šð Nð &.�Ô"à×#Ñ#Ð+Ü'5Ó'7×'GÑ'GÈÓ'P�Ô$å0à�J‰J×$Ñ$ WÐ-=Ñ%>Ô?à Ð-Ð-r(   c           	      ó  — i }|d   j                   j                  | _        | j                  |d<   g |d<   |D ]*  }|d   j                  |j                   j                  «       Œ, ||d<   |€9t	        |«      dk(  r t
        j                  j                  «       g}nt        d«      ‚|d   j                  d¬«      | _
        |d   |d	<   g |d
<   |D ]  }|d
   j                  |«       Œ | j                  |d<   t        j                  | j                  «      |d<   t        j                  | j                  «      |d<   || _        |d   | _        | j                   j#                  «        g }g }	ddlm}
  |
| j(                  «      }|j+                  || j                  | j(                  | j                  «       |j-                  ||||¬«      \  }}	||d<   |	|d<   |D ]×  }|j                   j                  }|j.                  €i n|j.                  }| j1                  «       |d<   | j3                  «       |d<   | j                  j4                  j7                  «       D ]  \  }}|s||vsŒ|||<   Œ ||_        t9        j:                  dt=        t?        |«      «      z   t=        |j.                  «      z   «       ŒÙ | j@                  €tC        «       jE                  |«      | _         ddl#m$} |jJ                  jM                  |d   «       ||	fS )Nr   r¸  r¹  r±  r   z0startup_program can't be None when loss is list.Frº  r¼  r½  ru   r¶  r¤  rÂ  rÄ  rÅ  rÆ  rÉ  rÊ  zfleet base opt info: rL   )'rË  r‰  r¸  rò   r$  rB   r^  rÌ  r,   rÍ  r¼  r8   rU   rV   r   rA   r¤  rÞ  rÙ  rÃ  rE   rÖ  Úminimize_losses_implrç  rk   rj   rè  rÈ   r   r–   ra   râ  r+   r   rß  rT   rM   r\   ré  )rG   ÚlossesÚstartup_programsr²  r³  rÌ   r±  r‰  rì  rí  rÃ  Úps_optimizerrû  rü  rý  rM   s                   r&   r°  zFleet._minimize_losses_implG  s  € ð ˆð $*¨!¡9§?¡?×#:Ñ#:ˆÔ Ø)-×)AÑ)AˆÐ%Ñ&Ø*,ˆÐ&Ñ'ÛˆDØÐ*Ñ+×2Ñ2°4·:±:×3EÑ3EÕFð à ˆ�‰àÐ#Ü�6‹{˜aÒÜ$*§M¡M×$IÑ$IÓ$KÐ#LÑ ä ØFóð ð '7°qÑ&9×&?Ñ&?ÈÐ&?Ó&OˆÔ#Ø,<¸QÑ,?ˆÐ(Ñ)Ø-/ˆÐ)Ñ*Û'ˆGØÐ-Ñ.×5Ñ5°gÕ>ð (ð !%× 0Ñ 0ˆ�Ñä+/¯=©=Ø×'Ñ'ó,
ˆÐ'Ñ(ô %)§M¡M°$×2MÑ2MÓ$NˆÐ Ñ!àˆŒà%Ð&6Ñ7ˆÔØ×Ñ×'Ñ'Ô)àˆØˆå=á/°×0KÑ0KÓLˆØ×$Ñ$ØØ×ÑØ×'Ñ'Ø×'Ñ'ô		
ð &2×%FÑ%FØÐ$ nÀ+ð &Gó &
Ñ"ˆ�lð +7ˆÐ&Ñ'Ø*6ˆÐ&Ñ'ãˆDØ—j‘j×(Ñ(ˆGØ$×/Ñ/Ð7‘r¸W×=OÑ=OˆHØ#'§?¡?Ó#4ˆH�ZÑ Ø#'×#4Ñ#4Ó#6ˆH�ZÑ ð ×,Ñ,×AÑA×GÑGÖIñØØá˜ Ò)Ø"#�H˜Q’Kð Jð "*ˆGÔÜ�K‰KØ'Ü”b˜“kÓ"ñ#ä�g×(Ñ(Ó)ñ*õð ð$ ×ÑÐ'Ü#1Ó#3×#CÑ#CÀGÓ#LˆDÔ å,à�
‰
× Ñ  Ð)9Ñ!:Ô;à˜\Ð)Ð)r(   )NFNÚINFO)F)Úsumrü   )NTr   )Nr   )l    (Ö\ )NNF)NN)NNN)Ar:   Ú
__module__Ú__qualname__Ú__doc__rH   r‡   r    r§   r«   r¯   r½   rÙ   rÜ   rh   rÄ   rþ   r  rj   rk   r	  r  r  r  r  r  r"  r&  r)  r+  r.  r2  r•   Úis_non_distributed_checkÚinited_runtime_handlerr;  r>  rA  rD  rG  rL  rQ  rU  rX  r[  rq  ru  rw  rz  r}  r‚  r…  r‹  r�  r‘  r˜  rš  r“  r¡  rŸ  r§  rª  r­  r´  r¯  r°  r3   r(   r&   r=   r=   d   sB  „ ñ7òrFð ØØØóEð^ óò>ò*ò.ò0(ðT +-°b¸dó :%ðx 02Àó "EòH?=òBòò3ò"0ò$.ò$0ò2ò8ò8ò-ò&2ó=ò*>ò"0ó$=ò,-ò&,ó"Cð$ Øò2ó ó ð2ð. Øò7ó ó ð7ò1ð Øñ,ó ó ð,ð Øñ;ó ó ð;ð0 Øñ<ó ó ð<ð. ØñCó ó ðCð. Øñ?ó ó ð?ð. Øñ+ó ó ð+ð. Øñ,ó ó ð,ð, ØØ!#¨2ò +ó ó ð+ðZ Øð Ø"Øò&
ó ó ð&
ðP Øò+
ó ó ð+
ðZ ØñJó ó ðJð ØñAó ó ðAð Øà9Cò
ó ó ð
ð ØñCó ó ðCð. Øà;?ò
ó ó ð
ð< Øò0ó ó ð0ó.ò`ò"0ð CHó:Qòxó"
ò3ò6ò7ð LPóCðL LPóh.ðZ	 ØØô\*r(   r=   )'rU   r_   r”   rB   Úpaddle.baser   Úpaddle.base.wrapped_decoratorr   Úpaddle.frameworkr   r   Úpaddle.framework.irr   Úbaser	   rf   Úbase.distributed_strategyr
   Úbase.meta_optimizer_factoryr   Úbase.role_makerr   r   Úbase.runtime_factoryr   Úbase.strategy_compilerr   Úmeta_parallelr   Úutils.log_utilr   r   Ú__all__r'   r5   r;   r	  r  r=   r3   r(   r&   Ú<module>r     sq   ðó Û 	Û ã Ý  Ý 8ß ;Ý 4å  Ý :Ý =ß @Ý 0Ý 4Ý 5ß 1à
€òò8	òñ& (Ð(@ÓAÐ Ù)Ð*DÓEÐ ÷*ò *r(   