Ë
    Ž\;j{O ã                   ó€   — d dl Z d dlZd dlmZ d dlmZmZ d dlZd dl	Z	d dl
mZmZ d dlmZmZmZmZ g Z G d„ d«      Zy)é    N)Údefaultdict)Ú
cmp_to_keyÚreduce)ÚcoreÚunique_name)Ú	ParameterÚProgramÚdefault_startup_programÚin_dygraph_modec                   ó  — e Zd ZdZd-d„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd.d„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	 d/d„Zd„ Z	 d.d„Z d „ Z!d!„ Z"d"„ Z#d#„ Z$d$„ Z%d%„ Z&d&„ Z'd'„ Z(d(„ Z)d)„ Z*d*„ Z+d+„ Z,	 d0d,„Z-y)1ÚPipelineOptimizera²
  
        :api_attr: Static Graph

    Pipeline Optimizer: Make a program to run as pipeline, that is splitting a
    program into multiple sections (sub-programs) and each section run on a
    device to enable the training of large scale models and the use of
    heterogeneous devices. Meanwhile, all sections run in the stype of pipeline.

    Args:
        optimizer (Optimizer): The optimizer to use, such as SGD.
        num_microbatches (int): Number of microbatches. [Optional. Default:1].
        start_cpu_core_id (int): The first cpu core id to use. [Optional. Default:0].

    Examples:
        .. code-block:: python

            >>> import paddle
            >>> import paddle.base as base
            >>> import paddle.base.layers as layers
            >>> import numpy as np

            >>> paddle.enable_static()
            >>> with base.device_guard("gpu:0"):
            ...     x = paddle.static.data(name='x', shape=[-1, 1], dtype='int64', lod_level=0)
            ...     y = paddle.static.data(name='y', shape=[-1, 1], dtype='int64', lod_level=0)
            ...     data_loader = base.io.DataLoader.from_generator(
            ...         feed_list=[x, y],
            ...         capacity=64,
            ...         use_double_buffer=True,
            ...         iterable=False)

            ...     emb_x = layers.embedding(input=x, param_attr=base.ParamAttr(name="embx"), size=[10,2], is_sparse=False)
            ...     emb_y = layers.embedding(input=y, param_attr=base.ParamAttr(name="emby",learning_rate=0.9), size=[10,2], is_sparse=False)

            >>> with base.device_guard("gpu:1"):
            ...     concat = layers.concat([emb_x, emb_y], axis=1)
            ...     fc = paddle.static.nn.fc(x=concat, name="fc", size=1, num_flatten_dims=1, bias_attr=False)
            ...     loss = paddle.mean(fc)
            >>> optimizer = paddle.optimizer.SGD(learning_rate=0.5)
            >>> optimizer = paddle.incubate.optimizer.PipelineOptimizer(optimizer)
            >>> optimizer.minimize(loss)

            >>> def train_reader():
            ...     for _ in range(4):
            ...         x = np.random.random(size=[1]).astype('int64')
            ...         y = np.random.random(size=[1]).astype('int64')
            ...         yield x, y
            >>> data_loader.set_sample_generator(train_reader, batch_size=1)

            >>> place = paddle.CUDAPlace(0)
            >>> exe = paddle.static.Executor(place)
            >>> exe.run(paddle.static.default_startup_program())
            >>> batch_size = 1
            >>> data_loader.start()
            >>> exe.train_from_dataset(
            ...         paddle.static.default_main_program())
            >>> data_loader.reset()
    c                 óp  — d| _         t        j                  «       rd| _         t        «       rt	        d«      ‚t
        j                  j                  t
        j                  j                  j                  j                  f}t        ||«      st        d|› dt        |«      › d�«      ‚|| _        | j                  | _        t#        | j                   d«      r2| j                   j$                  | _        t#        | j                   d«      rŒ2|dk\  sJ d	«       ‚|| _        |d
k\  sJ d«       ‚|| _        d | _        t        j,                  }|j.                  | _        |j3                  «       | _        |j7                  «       | _        |j;                  «       | _        d | _        g | _         i | _!        d | _"        d | _#        y )NÚcpuÚgpuz,In dygraph, don't support PipelineOptimizer.zGThe 'optimizer' parameter for PipelineOptimizer must be an instance of z, but the given type is Ú.Ú	inner_opté   z*num_microbatches must be a positive value.r   z1start_cpu_core_id must be a non-negative integer.)$Ú_devicer   Úis_compiled_with_cudar   Ú	ExceptionÚpaddleÚ	optimizerÚ	OptimizerÚstaticÚampÚ	decoratorÚOptimizerWithMixedPrecisionÚ
isinstanceÚ
ValueErrorÚtypeÚ
_optimizerÚ_origin_optimizerÚhasattrr   Ú_num_microbatchesÚ_start_cpu_core_idÚ_place_listÚop_proto_and_checker_makerÚOpRoleÚ_op_roleÚkOpRoleAttrNameÚ_op_role_keyÚkOpRoleVarAttrNameÚ_op_role_var_keyÚkOpDeviceAttrNameÚ_op_device_keyÚ_param_device_mapÚ_pipeline_pairÚ_pp_ring_mapÚoutput_var_to_opÚinput_var_to_op)Úselfr   Únum_microbatchesÚstart_cpu_core_idÚvalid_optimizersÚop_makers         úkG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/incubate/optimizer/pipeline.pyÚ__init__zPipelineOptimizer.__init__^   sŸ  € ØˆŒÜ×%Ñ%Ô'Ø ˆDŒLÜÔÜÐJÓKÐKä×Ñ×&Ñ&Ü�M‰M×Ñ×'Ñ'×CÑCð
Ðô ˜)Ð%5Ô6Üð<à#Ð$Ð$<¼TÀ)»_Ð<MÈQðPóð ð
 $ˆŒð "&§¡ˆÔÜ�d×,Ñ,¨kÔ:Ø%)×%;Ñ%;×%EÑ%EˆDÔ"ô �d×,Ñ,¨kÕ:ð  Ò!ð	8à7ó	8Ø!à!1ˆÔà Ò"ð	?à>ó	?Ø"à"3ˆÔØˆÔÜ×2Ñ2ˆØ Ÿ™ˆŒØ$×4Ñ4Ó6ˆÔØ (× ;Ñ ;Ó =ˆÔØ&×8Ñ8Ó:ˆÔØ!%ˆÔØ ˆÔØˆÔØ $ˆÔØ#ˆÕó    c                 óš  — |j                   |   }|j                  j                  «       d   }|j                  |«      }d}|j                  dk(  rŠt        j                  |dz   «      }|j                  |dgd¬«      }|j                  |dz   |z   dd|id	|id
|j                  d|j                  | j                  | j                  j                  i¬«       |dz  }|j                  |dz   |z   |j                  dk(  rdndd|j                  dk(  rn|id	|j                  dk(  rn|id| j                  | j                  | j                  j                  ddi¬«       |dz  }|j                  dk(  r]|j                  |dz   |z   ddid	|id
|j                  d|j                  | j                  | j                  j                  i¬«       |dz  }|S )zj
        Insert allreduce op to sync global information for global
        gradient clip and amp.
        r   Ú
reduce_anyÚ_cast_int32r   Úint32)ÚnameÚshapeÚdtypeÚcastÚXÚOutÚin_dtypeÚ	out_dtype©r    ÚinputsÚoutputsÚattrsÚc_allreduce_maxÚc_allreduce_sumÚring_idÚuse_calc_streamT)ÚopsÚdescÚoutput_arg_namesÚvarr    r   ÚgenerateÚ
create_varÚ
_insert_oprC   r+   r)   ÚOptimizeÚglobal_ring_id)	r5   Úop_idxÚblockÚopÚout_nameÚout_varÚoffsetÚtemp_var_nameÚtemp_vars	            r:   Ú_insert_allreduce_opz&PipelineOptimizer._insert_allreduce_op‹   sð  € ð
 �Y‰Y�vÑˆØ—7‘7×+Ñ+Ó-¨aÑ0ˆØ—)‘)˜HÓ%ˆØˆØ�7‰7�lÒ"ä'×0Ñ0°¸MÑ1IÓJˆMØ×'Ñ'Ø"¨1¨#°Wð (ó ˆHð ×ÑØ˜‘
˜VÑ#ØØ˜W�~Ø Ð)à §¡Ø §¡Ø×%Ñ% t§}¡}×'=Ñ'=ðð ô 
ð �a‰KˆFØ×ÑØ�Q‰J˜Ñà�w‰w˜,Ò&ñ #à"Ø R§W¡W°Ò%<™À'ÐJØ¨¯©°<Ò(?™HÀWÐMà˜4×.Ñ.Ø×!Ñ! 4§=¡=×#9Ñ#9Ø! 4ðð 	ô 	
ð 	�!‰ˆØ�7‰7�lÒ"Ø×ÑØ˜‘
˜VÑ#ØØ˜X�Ø Ð(à §¡Ø §¡Ø×%Ñ% t§}¡}×'=Ñ'=ðð ô 
ð �a‰KˆFØˆr<   c                 óR	  — t        «       }d}d}|j                  j                  «       }|||z   k  �rfd}|j                  |   }g }	|j                  dk(  r| j                  |«      rd}�n>|j                  dk(  rr| j                  |«      ra|j                  j                  d«      D ]%  }
|j                  |
«      sŒ|	j                  |
«       Œ' |j                  j                  d|	«       �n½|j                  dk(  r}|j                  j                  d«      D ]%  }
|j                  |
«      sŒ|	j                  |
«       Œ' |j                  j                  d|	«       |j                  j                  d|	«       �n1|j                  d	k(  r¡|j                  j                  d«      D ]%  }
|j                  |
«      sŒ|	j                  |
«       Œ' |j                  j                  d|	«       |j                  j                  d|	«       t        |	«      dk(  r™|j                  |«       |d
z  }�Œý|j                  dk(  rr| j                  |«      ra|j                  j                  d«      D ]%  }
|j                  |
«      sŒ|	j                  |
«       Œ' |j                  j                  d|	«       d}|j                  j                  «       |j                  j                  «       z   }|D �]x  }||v sd|v rŒ|j!                  |«       |j                  t#        |«      «      rŒ9|j%                  t#        |«      «      }|j                  t&        j(                  j*                  j,                  k(  rA|j/                  |t&        j(                  j*                  j,                  |j0                  ¬«      }n¢t3        |t4        «      r€|j7                  |j8                  |j:                  |j<                  |j                  |j>                  |j@                  |jB                  |jD                  |jF                  |jH                  ¬«
      }n|jK                  |d«      }| jM                  ||«       �Œ{ |d
z  }| jN                  s|s�ŒF| jQ                  |d
z
  |«      }||z  }||z  }|||z   k  r�Œf|jS                  «        y )Nr   Fr>   TÚconcatrE   Úupdate_loss_scalingrF   Úcheck_finite_and_unscaler   ÚsumÚ_blocking_queue)rA   r    Úpersistable)
rA   rB   rC   r    Ú	lod_levelÚstop_gradientÚ	trainableÚoptimize_attrÚregularizerÚ
error_clip)*ÚsetrR   Úop_sizerQ   r    Ú_is_optimize_opÚinputÚ_find_var_recursiveÚappendÚ	set_inputÚ
set_outputÚlenÚ
_remove_opÚ_is_gradient_clip_opÚinput_arg_namesrS   ÚaddÚstrÚ_var_recursiver   ÚVarDescÚVarTypeÚREADERrV   ri   r   r   Úcreate_parameterrA   rB   rC   rj   rk   rl   rm   rn   ro   Ú_clone_variableÚ_clone_var_attrÚuse_shardingrb   Ú_sync_with_cpp)r5   r[   Ú	ori_blockÚused_var_setÚadded_op_numrZ   rq   Úshould_insertr\   Ú
reserved_xÚ
input_nameÚvarsrT   Ú
source_varÚdest_varÚinserted_opss                   r:   Ú_create_varszPipelineOptimizer._create_varsÃ   s÷  € ä“uˆØˆØˆØ—*‘*×$Ñ$Ó&ˆØ�w Ñ-Ó-ð "ˆMØ—‘˜6Ñ"ˆBð ˆJØ�w‰w˜,Ò&¨4×+?Ñ+?ÀÔ+CØ $’Ø—‘˜HÒ$¨×)=Ñ)=¸bÔ)AØ"$§'¡'§-¡-°Ö"4�JØ×0Ñ0°Õ<Ø"×)Ñ)¨*Õ5ð #5ð —‘×!Ñ! # zÖ2Ø—‘Ð1Ò1Ø"$§'¡'§-¡-°Ö"4�JØ×0Ñ0°Õ<Ø"×)Ñ)¨*Õ5ð #5ð —‘×!Ñ! # zÔ2Ø—‘×"Ñ" 5¨*Ö5Ø—‘Ð6Ò6Ø"$§'¡'§-¡-°Ö"4�JØ×0Ñ0°Õ<Ø"×)Ñ)¨*Õ5ð #5ð —‘×!Ñ! # zÔ2Ø—‘×"Ñ" 5¨*Ô5Ü�z“? aÒ'Ø×$Ñ$ VÔ,Ø˜q‘L�GÙØ—‘˜EÒ! d×&?Ñ&?ÀÔ&CØ"$§'¡'§-¡-°Ö"4�JØ×0Ñ0°Õ<Ø"×)Ñ)¨*Õ5ð #5ð —‘×!Ñ! # zÔ2Ø $�à—7‘7×*Ñ*Ó,¨r¯w©w×/GÑ/GÓ/IÑIˆDÜ�ð ˜,Ñ&Ð*;¸sÑ*BØØ× Ñ  Ô%Ø×,Ñ,¬S°«XÔ6ØØ&×5Ñ5´c¸#³hÓ?�
Ø—?‘?¤d§l¡l×&:Ñ&:×&AÑ&AÒAØ$×/Ñ/Ø Ü!Ÿ\™\×1Ñ1×8Ñ8Ø$.×$:Ñ$:ð  0ó  ‘Hô
   
¬IÔ6Ø$×5Ñ5Ø'Ÿ_™_Ø(×.Ñ.Ø(×.Ñ.Ø'Ÿ_™_Ø",×"6Ñ"6Ø&0×&>Ñ&>Ø",×"6Ñ"6Ø&0×&>Ñ&>Ø$.×$:Ñ$:Ø#-×#8Ñ#8ð  6ó  ‘Hð  %×4Ñ4°ZÀÓG�HØ×$Ñ$ X¨zÖ:ð= ðB �a‰KˆFØ× Ò ©ÙØ×4Ñ4°V¸a±ZÀÓGˆLØ˜LÑ(ˆLØ�lÑ"ˆFð] �w Ñ-Ô-ð^ 	×ÑÕr<   c                 ó
  — | j                   |j                  v sJ ‚t        |j                  | j                   «      «      }|t        | j                  j
                  «      z  xr" |t        | j                  j                  «      z  S ©N)r+   Ú
attr_namesÚintÚattrr)   ÚBackwardÚLoss)r5   r\   Úop_roles      r:   Ú_is_loss_grad_opz"PipelineOptimizer._is_loss_grad_op  sp   € Ø× Ñ  B§M¡MÑ1Ð1Ð1Ü�b—g‘g˜d×/Ñ/Ó0Ó1ˆØœ˜TŸ]™]×3Ñ3Ó4Ñ4ò 
¸Ä3Ø�M‰M×ÑóD
ñ :
ð 	
r<   c                 ó¾   — | j                   |j                  v xrD t        |j                  | j                   «      «      t        | j                  j
                  «      k(  S r“   )r+   r”   r•   r–   r)   ÚForward©r5   r\   s     r:   Ú_is_forward_opz PipelineOptimizer._is_forward_op!  sI   € Ø× Ñ  B§M¡MÐ1ò 
Ü�—‘˜×)Ñ)Ó*Ó+¬s°4·=±=×3HÑ3HÓ/IÑIð	
r<   c                 ó¾   — | j                   |j                  v xrD t        |j                  | j                   «      «      t        | j                  j
                  «      z  S r“   )r+   r”   r•   r–   r)   r—   r�   s     r:   Ú_is_backward_opz!PipelineOptimizer._is_backward_op&  óI   € Ø× Ñ  B§M¡MÐ1ò 
Ü�—‘˜×)Ñ)Ó*Ó+¬c°$·-±-×2HÑ2HÓ.IÑIð	
r<   c                 ó¾   — | j                   |j                  v sJ ‚t        |j                  | j                   «      «      t        | j                  j
                  «      k(  S r“   )r+   r”   r•   r–   r)   r˜   r�   s     r:   Ú_is_loss_opzPipelineOptimizer._is_loss_op+  sH   € Ø× Ñ  B§M¡MÑ1Ð1Ð1Ü�2—7‘7˜4×,Ñ,Ó-Ó.´#°d·m±m×6HÑ6HÓ2IÑIÐIr<   c                 ó¾   — | j                   |j                  v xrD t        |j                  | j                   «      «      t        | j                  j
                  «      z  S r“   )r+   r”   r•   r–   r)   rX   r�   s     r:   rr   z!PipelineOptimizer._is_optimize_op/  r¡   r<   c                 ó^   — d|j                   v xr d|j                   v xr d|j                   v S )NÚParamÚGradÚLearningRate)Úinput_namesr�   s     r:   Ú_is_update_opzPipelineOptimizer._is_update_op4  s5   € à�r—~‘~Ð%ò 3Ø˜"Ÿ.™.Ð(ò3à 2§>¡>Ð1ð	
r<   c                 óÌ  — t        t        «      }|j                  d«      }|j                  D �]  }|j	                  | j
                  «      }|| j                  › d�k(  rn|D ]h  }||   }|j                  }|j                  «       j                  j                  «       }	|	j                  |«       |	j                  | j
                  d«       Œj ŒŸ||   }|j                  }|j                  «       j                  j                  «       }	|	j                  |«       |	j                  | j
                  d«       �Œ g }
|D ](  }||   }|j                  «        |
j                  |«       Œ* |
S )a  
        Split a program into sections according to devices that ops run on.
        The op whose op_device attr is "gpu:all" is copied to all sections.

        Args:
            main_program (Program): the main program
            devices: all used devices
        r   ú:allÚ )r   r	   r[   rQ   r–   r/   r   rR   Úglobal_blockÚ	append_opÚ	copy_fromÚ	_set_attrr†   ru   )r5   Úmain_programÚdevicesÚdevice_program_mapr[   r\   ÚdeviceÚprogramÚop_descÚap_opÚprogram_listÚkeys               r:   Ú_split_programz PipelineOptimizer._split_program;  s=  € ô )¬Ó1Ðà×"Ñ" 1Ó%ˆØ—)•)ˆBØ—W‘W˜T×0Ñ0Ó1ˆFà˜DŸL™L˜>¨Ð.Ò.Û%�FØ0°Ñ8�GØ Ÿg™g�GØ#×0Ñ0Ó2×7Ñ7×AÑAÓC�EØ—O‘O GÔ,Ø—O‘O D×$7Ñ$7¸Õ<ñ &ð -¨VÑ4�ØŸ'™'�Ø×,Ñ,Ó.×3Ñ3×=Ñ=Ó?�Ø—‘ Ô(Ø—‘ × 3Ñ 3°RÖ8ð ð" ˆÛˆCØ(¨Ñ-ˆGØ×"Ñ"Ô$Ø×Ñ Õ(ð ð
 Ðr<   c                 ój   — d|v sd|v sJ d«       ‚|d|j                  d«       }| j                  |   }|S )a  
        For adam optimizer, it will add accumulators and initialize them
        with fill_constant, and force the op device to cpu. Hence, we should
        get the real op_device attribute of the fill_constant as the device
        where the corresponding parameters on.
        Úbeta1_pow_accÚbeta2_pow_acczPFor accumulators for Adam, the name must contain beta1_pow_acc or beta2_pow_acc.r   Ú_beta)Úindexr0   )r5   Úvar_nameÚ
param_namerµ   s       r:   Ú"_get_op_device_for_startup_programz4PipelineOptimizer._get_op_device_for_startup_programa  sQ   € ð  (Ñ*¨oÀÑ.Ið 	
ð ó	
ÐIð ˜a (§.¡.°Ó"9Ð:ˆ
Ø×'Ñ'¨
Ñ3ˆØˆr<   c                 ó~  — |j                  «       }t        «       }|j                  D ]ã  }|j                  | j                  «      }|dk(  r6|j
                  dk(  sJ d«       ‚|j                  d   }| j                  |«      }|rt        |j                  d«      d   «      }nd }|r|k7  rŒƒ|j                  }	|j                  «       j                  j                  «       }
|
j                  |	«       |
j                  | j                  d«       Œå |j                  «        | j                  |j                  «       |«       |S )Nr   Úfill_constantzcFor ops in startup program with the op_device attribute of cpu, they must be of type fill_constant.r   Ú:r   r­   )r®   r	   rQ   r–   r/   r    rS   rÃ   r•   ÚsplitrR   r¯   r°   r±   r†   r‘   )r5   Ústartup_programÚ	device_idr[   Únew_startup_programr\   rµ   Ú
output_varÚdevice_indexr·   r¸   s              r:   Ú_split_startup_programz(PipelineOptimizer._split_startup_programp  s%  € Ø×,Ñ,Ó.ˆÜ%›iÐØ—)”)ˆBØ—W‘W˜T×0Ñ0Ó1ˆFØ˜ŠØ—w‘w /Ò1ð ðBóÐ1ð  ×0Ñ0°Ñ3�
Ø×@Ñ@ÀÓL�áÜ" 6§<¡<°Ó#4°QÑ#7Ó8‘ð �Ù˜,¨)Ò3ØØ—g‘gˆGØ'×4Ñ4Ó6×;Ñ;×EÑEÓGˆEØ�O‰O˜GÔ$Ø�O‰O˜D×/Ñ/°Õ4ð) ð* 	×*Ñ*Ô,Ø×ÑÐ-×:Ñ:Ó<¸eÔDØ"Ð"r<   c                 óÀ   — d|v r|j                  dd«      }d|v r|j                  dd«      }| j                  |   }|€yd}t        |«      D ]  \  }}||kD  sŒ|} |S  |S )zM
        Find the post op that has variable named var_name as input.
        z
.cast_fp32r­   ú
.cast_fp16N)Úreplacer4   Úreversed)r5   rÀ   rÁ   Úpost_opsÚ	result_opÚpost_opÚpost_idxs          r:   Ú_find_post_opzPipelineOptimizer._find_post_opŒ  sˆ   € ð
 ˜8Ñ#Ø×'Ñ'¨°bÓ9ˆHØ˜8Ñ#Ø×'Ñ'¨°bÓ9ˆHà×'Ñ'¨Ñ1ˆØÐØØˆ	Ü!)¨(Ö!3ÑˆG�XØ˜%ÓØ#�	ØØÐð	 "4ð Ðr<   c                 óh   — | j                   |   }|€yd}t        |«      D ]  \  }}||k  sŒ|} |S  |S )ze
        Find the previous op of op with index that outputs
        variable named var_name.
        N)r3   rÑ   )r5   rÀ   rÁ   Úprev_opsrÓ   Úprev_opÚprev_idxs          r:   Ú_find_prev_opzPipelineOptimizer._find_prev_op   sT   € ð
 ×(Ñ(¨Ñ2ˆØÐØØˆ	Ü!)¨(Ö!3ÑˆG�XØ˜%ÓØ#�	ØØÐð	 "4ð Ðr<   c                 óL   — |j                  ||«       |j                  ||«       y r“   )Ú_rename_inputÚ_rename_output)r5   r\   Úold_nameÚnew_names       r:   Ú_rename_argzPipelineOptimizer._rename_arg¯  s"   € Ø
×Ñ˜ 8Ô,Ø
×Ñ˜( HÕ-r<   Nc                 ó  — |j                  ||j                  |€|j                  n||j                  |j                  |j
                  |j                  |j                  j                  «       ¬«      }| j                  ||«       |S )z—
        Create a new var for block, which has the same type,
        shape and dtype as ref_var, then rename it with the
        name `name`.
        )rA   rB   rC   r    rj   ri   Úis_dataÚneed_check_feed)
rV   rB   rC   r    rj   ri   rã   rR   rä   r„   )r5   r[   Úref_varrA   rC   Únew_vars         r:   Ú_create_varzPipelineOptimizer._create_var³  sw   € ð ×"Ñ"ØØ—-‘-Ø#( =�'—-’-°eØ—‘Ø×'Ñ'Ø×+Ñ+Ø—O‘OØ#ŸL™L×8Ñ8Ó:ð #ó 	
ˆð 	×Ñ˜W gÔ.Øˆr<   c                 ób   — |j                   |_         t        |d«      r|j                  |_        y y )NÚis_distributed)rk   r#   ré   )r5   ÚdestÚsrcs      r:   r„   z!PipelineOptimizer._clone_var_attrÆ  s.   € Ø ×.Ñ.ˆÔÜ�3Ð(Ô)Ø"%×"4Ñ"4ˆDÕð *r<   c                 ó`   — |j                  t        j                  «       «      }|dk7  r|d| S |S )zD
        Strip the grad suffix from the given variable name
        éÿÿÿÿN)Úfindr   Úgrad_var_suffix)r5   rA   Úposs      r:   Ú_strip_grad_suffixz$PipelineOptimizer._strip_grad_suffixË  s2   € ð �i‰iœ×,Ñ,Ó.Ó/ˆØ  BšYˆt�D�SˆzÐ0¨DÐ0r<   c                 ó0   — |t        j                  «       z   S )z?
        Append grad suffix to the given variable name
        )r   rï   )r5   rA   s     r:   Ú_append_grad_suffixz%PipelineOptimizer._append_grad_suffixÒ  s   € ð ”d×*Ñ*Ó,Ñ,Ð,r<   c                 ó˜   — |j                  | j                  «      r|j                  | j                  «      nd}|r|dd dk(  sJ d«       ‚|S )z6
        Get the op_device attribute of a op.
        Nr   é   r   z<Now, only gpu devices are supported in pipeline parallemism.)Úhas_attrr/   r–   )r5   r\   rµ   s      r:   Ú_get_op_device_attrz%PipelineOptimizer._get_op_device_attrØ  s^   € ð �{‰{˜4×.Ñ.Ô/ð �G‰G�D×'Ñ'Ô(àð 	ñ
 Ø˜!˜A�; %Ò'ð ð5óÐ'ð ˆr<   c                 óâ  — t        | j                  j                  «      }|j                  | j                  «      |k(  r*|j                  | j                  | j                  › d�«       y|j                  dk(  rú| j                  |«      ré|j                  j                  «       D ]  }d|v rŒJ d«       ‚ t        |j                  j                  «       «      dk(  sJ ‚|j                  j                  «       d   }| j                  ||«      }|j                  d«      sJ |j                  › d|› �«       ‚|j                  | j                  «      }|sJ d	«       ‚|j                  | j                  |«       y|j                  d
k(  s|j                  dk(  r†| j                  |«      s| j!                  |«      rd| j#                  ||j                  j%                  d«      d   «      }	|j                  | j                  |	j                  | j                  «      «       y|j                  dk(  �r| j'                  |«      �s t        |j                  «      dk(  rt        |j                  «      dk(  sJ ‚|j                  d   }
|j                  d   }d|v rH| j                  ||«      }|j                  | j                  |j                  | j                  «      «       y| j#                  ||j                  j%                  d«      d   «      }	|j                  | j                  |	j                  | j                  «      «       y| j)                  |«      �r&d}|j*                  ||z      j                  | j                  «      r+|j*                  ||z      j                  | j                  «      s]|dz  }|j*                  ||z      j                  | j                  «      sŒ1|j*                  ||z      j                  | j                  «      sŒ]|j*                  ||z      j                  | j                  «      }|sJ d«       ‚t-        |«      D ].  }|j*                  ||z      j                  | j                  |«       Œ0 y| j'                  |«      rp|j                  d
k(  ra|j/                  d«      }t        |«      dk(  sJ ‚| j1                  |d   «      }| j2                  |   }|j                  | j                  |«       y| j5                  |«      s| j7                  |«      rÚ| j8                  |j:                  v sJ d«       ‚|j                  | j8                  «      }t        |«      dk(  sJ d«       ‚|d   }| j2                  |   }|j                  dk(  s<|j                  dk(  s-|j                  dk(  s|j                  dk(  s|j                  dk(  r| j                  › d�}|j                  | j                  |«       y|j                  dk(  s|j                  dk(  r�|j                  | j                  | j                  › d�«       |j                  | j                  | j                  j                  «       |j                  d   }|j=                  |«      }d|_        yg d¢}|j                  |v sJ d|› d|j                  › �«       ‚| j'                  |«      sJ ‚|j                  | j                  | j                  › d�«       y)a  
        Add op_device attrribute for ops that have not that attribute set.
        We use "gpu:all" to represent the op should be put on all
        sub-programs, such as lr-related ops. Note that: "gpu:all"
        is only used by pipeline as an indicator.
        r¬   rg   z@RENAME@z3The op must be sum used to accumulate renamed vars.r   r   Ú	op_devicez has no op_device attr for var z$The post op must have op_device set.rD   ÚscalerE   Úmemcpyz@Fetchz1Please put you program within device_guard scope.rF   zEgradient_clip and regularization ops must have op_role_var attribute.é   zHop_role_var for gradient_clip regularization ops must have two elements.ÚsqrtrÅ   Úelementwise_maxÚelementwise_divÚalloc_float_statusÚclear_float_statusT)re   r>   rd   rg   rf   rû   z9For other ops without op_device set, they must be one of z, but it is N) r•   r)   ÚLRSchedr–   r+   r±   r/   r   r    r    rR   r{   rx   rS   rÖ   rö   rž   rÛ   rs   rr   r£   rQ   ÚrangeÚoutputrñ   r0   rz   Ú_is_regularization_opr-   r”   rT   ri   )r5   r\   Úidxr[   Úlrsched_rolerA   r]   rÔ   rµ   rÙ   rŒ   Úoutput_namer_   ÚiÚ	grad_namerÂ   Úop_role_varÚfloat_status_nameÚfloat_status_varÚother_known_opss                       r:   Ú_add_op_device_attr_for_opz,PipelineOptimizer._add_op_device_attr_for_opè  s,  € ô ˜4Ÿ=™=×0Ñ0Ó1ˆØ�7‰7�4×$Ñ$Ó%¨Ò5ð �L‰L˜×,Ñ,°·±°¸dÐ.CÕDà�W‰W˜Ò $×"6Ñ"6°rÔ":àŸ™×/Ñ/Ö1�à $Ò&ðIàHóIØ&ð 2ô �r—w‘w×/Ñ/Ó1Ó2°aÒ7Ð7Ð7Ø—w‘w×/Ñ/Ó1°!Ñ4ˆHØ×(Ñ(¨¨hÓ7ˆGØ×#Ñ#Øôð Jà—,‘,�Ð>¸x¸jÐIóJð ð —\‘\ $×"5Ñ"5Ó6ˆFÙÐAÐAÓA�6Ø�L‰L˜×,Ñ,¨fÕ5Ø�g‰g˜Ò 2§7¡7¨gÒ#5Ø× Ñ  Ô$¨×(;Ñ(;¸BÔ(?à×(Ñ(¨¨b¯g©g¯m©m¸CÓ.@ÀÑ.CÓDˆGØ�L‰L˜×,Ñ,¨g¯l©l¸4×;NÑ;NÓ.OÕPØ�W‰W˜Ó ¨×)=Ñ)=¸bÕ)Aô �B×&Ñ&Ó'¨1Ò,´°R×5HÑ5HÓ1IÈQÒ1NðØNà×+Ñ+¨AÑ.ˆJØ×-Ñ-¨aÑ0ˆKØ˜;Ñ&Ø×,Ñ,¨S°+Ó>�Ø—‘Ø×'Ñ'¨¯©°d×6IÑ6IÓ)Jõð ×,Ñ,¨S°"·'±'·-±-ÀÓ2DÀQÑ2GÓH�Ø—‘Ø×'Ñ'¨¯©°d×6IÑ6IÓ)Jõð ×Ñ˜bÕ!àˆFØ—i‘i  f¡Ñ-×6Ñ6Ø×#Ñ#ôà—Y‘Y˜s V™|Ñ,×1Ñ1°$×2EÑ2EÔFØ˜!‘�ð —i‘i  f¡Ñ-×6Ñ6Ø×#Ñ#õà—Y‘Y˜s V™|Ñ,×1Ñ1°$×2EÑ2EÕFà—Y‘Y˜s V™|Ñ,×1Ñ1°$×2EÑ2EÓFˆFÙÐNÐNÓN�6Ü˜6–]�Ø—	‘	˜# ™'Ñ"×,Ñ,¨T×-@Ñ-@À&ÕIñ #à×!Ñ! "Ô%¨"¯'©'°VÒ*;àŸ	™	 %Ó(ˆIÜ�y“> QÒ&Ð&Ð&Ø×0Ñ0°¸1±Ó>ˆJØ×+Ñ+¨JÑ7ˆFØ�L‰L˜×,Ñ,¨fÕ5Ø×&Ñ& rÔ*¨d×.HÑ.HÈÔ.Lð ×(Ñ(¨B¯M©MÑ9ð ðJóÐ9ð Ÿ'™' $×"7Ñ"7Ó8ˆKÜ�{Ó# qÒ(ð ð=óÐ(ð % Q™ˆJØ×+Ñ+¨JÑ7ˆFð —‘˜5Ò Ø—7‘7˜fÒ$Ø—7‘7˜oÒ-Ø—7‘7Ð/Ò/Ø—7‘7Ð/Ò/à ŸL™L˜>¨Ð.�Ø�L‰L˜×,Ñ,¨fÕ5Ø�W‰WÐ,Ò,°·±Ð;OÒ0OØ�L‰L˜×,Ñ,°·±°¸dÐ.CÔDð �L‰L˜×*Ñ*¨D¯M©M×,AÑ,AÔBà "× 3Ñ 3°AÑ 6ÐØ$Ÿy™yÐ):Ó;Ðð ,0ÐÕ(òˆOð —7‘7˜oÑ-ð ð6Ø6EÐ5Fð GØ—g‘g�Yð óÐ-ð
 ×'Ñ'¨Ô+Ð+Ð+Ø�L‰L˜×,Ñ,°·±°¸dÐ.CÕDr<   c                 óH  — t        t        |j                  «      «      D ]�  \  }}|j                  dk(  s|j                  dk(  s|j                  dk(  r*|j	                  | j
                  | j                  › d�«       Œ]| j                  |«      rŒo| j                  |||«       Œƒ y)ze
        Add op_device attrribute for ops in block that have
        not that attribute set.
        Úcreate_py_readerÚreadÚcreate_double_buffer_readerr¬   N)	Ú	enumerateÚlistrQ   r    r±   r/   r   r÷   r  )r5   r[   r  r\   s       r:   Ú_add_op_device_attrz%PipelineOptimizer._add_op_device_attrb  sŒ   € ô
 !¤ e§i¡i£Ö1‰GˆC�à—‘Ð-Ò-Ø—7‘7˜fÒ$Ø—7‘7Ð;Ò;ð —‘˜T×0Ñ0°T·\±\°NÀ$Ð2GÔHØà×'Ñ'¨Ô+ØØ×+Ñ+¨B°°UÕ;ñ! 2r<   c           	      ó6  — g }t        | j                  j                  «      t        | j                  j                  «      t        | j                  j                  «      t        | j                  j
                  «      t        | j                  j                  «      t        | j                  j                  «      t        | j                  j
                  «      z  g}|j                  D �]°  }|j                  |j                  «      sQ|j                  dk(  r;|j                  | j                  «      t        | j                  j                  «      k(  sJ d«       ‚|j                  | j                  «      s"J d|j                  › d| j                  › d�«       ‚|j                  | j                  «      }t        |«      |v s"J dj                  ||j                  |«      «       ‚|j                  | j                  «      s"J d|j                  › d| j                  › d�«       ‚|j                  | j                  «      }|sJ d|j                  › d�«       ‚|| j                  › d	�k(  r�Œz|j!                  d
«      d   }|dk(  sJ d«       ‚||vs�Œ |j#                  |«       �Œ³ |S )z™
        Check whether ops in a block have both the op_device and the
        op_role attributes set.
        Then, return all devices in order.
        Úconditional_blockz`Now, the only supported op without kernel is conditional_block, and its op role must be LRSched.zop (z	) has no z attribute.z&op_role {} for op {} must be one of {}zop_device attribute for op z has not been set.r¬   rÆ   r   r   z<Now only gpu devices are supported for pipeline parallelism.)r•   r)   r  rœ   r—   r˜   rX   rQ   Ú_has_kernelr    r–   r+   rö   Úformatr/   r   rÇ   ru   )r5   r[   Údevice_listÚvalid_op_role_valuer\   r™   rµ   Údev_types           r:   Ú_check_validationz#PipelineOptimizer._check_validationy  sl  € ð ˆô �—‘×%Ñ%Ó&Ü�—‘×%Ñ%Ó&Ü�—‘×&Ñ&Ó'Ü�—‘×"Ñ"Ó#Ü�—‘×&Ñ&Ó'Ü�—‘×&Ñ&Ó'¬#¨d¯m©m×.@Ñ.@Ó*AÑAð
Ðð —)•)ˆBØ—>‘> "§'¡'Ô*Ø—w‘wÐ"5Ò5Ø—G‘G˜D×-Ñ-Ó.´#°d·m±m×6KÑ6KÓ2LÒLððJóð ð —;‘;Ø×!Ñ!ôð Gà�b—g‘g�Y˜i¨×(9Ñ(9Ð':¸+ÐFóGð ð —g‘g˜d×/Ñ/Ó0ˆGä�G“Ð 3Ñ3ðà7×>Ñ>Ø˜Ÿ™Ð"5óóØ3ð
 —;‘;Ø×#Ñ#ôð Ià�b—g‘g�Y˜i¨×(;Ñ(;Ð'<¸KÐHóIð ð —W‘W˜T×0Ñ0Ó1ˆFÙð Ø-°·±°	Ð9KÐLó�6ð ˜DŸL™L˜>¨Ð.Ò.Ùà—|‘| CÓ(¨Ñ+ˆHØ˜uÒ$ð ð,óÐ$ð
 ˜[Ó(Ø×"Ñ" 6Ö*ðI ðL Ðr<   c                 ó¨  ‡ ‡‡‡‡	‡
‡‡‡‡— i Šd}t        t        ‰j                  «      «      D ]  \  ŠŠ‰ j                  ‰«      sŒ‰} n d|dœŠ
t        t        ‰j                  «      «      D �]R  \  ŠŠ‰j	                  ‰ j
                  «      }|‰ j                  › d�k(  rŒ5‰j                  D �]  Š‰j                  ‰«      }|j                  rŒ"d}‰ j                  ‰‰«      }|€‰‰ j                  vrŒG‰ j                  ‰   }|s|r|j	                  ‰ j
                  «      nd}|�|‰ j                  › d�k(  rŒŒ||k(  rŒ’‰‰vrg ‰‰<   ||f‰‰   v rŒ¥|j                  d«      d   dz   Š	ˆˆ fd„Šˆˆˆˆ	ˆ
ˆˆˆˆ ˆf
d„Š ‰t        |j                  d«      d   «      t        |j                  d«      d   «      «       �Œ �ŒU ‰j                  «        y)	zp
        Insert a pair of send and recv ops for every two
        consecutive ops on different devices.
        Nr   )rÀ   Úfirst_optimize_indexr¬   rÆ   c                 óæ   •— ‰j                  ‰«      }‰j                  ‰«      }|s|s
J d‰› �«       ‚|r|| k  sJ dj                  || ‰«      «       ‚y |r|| kD  sJ dj                  || ‰«      «       ‚y y )Nzdsend/recv in pipeline should only be inserted in forward or backward,please check the op_role of op=z€In forward, send/recv can only be passed forward, but now prev_stage={} great than cur_stage={}, please check op_device of op={}z�In backward, send/recv can only be passed backward, but now prev_stage={} less than cur_stage={}, please check op_device of op={})rž   r    r  )Úcur_idÚprev_idÚ
is_forwardÚis_backwardr\   r5   s       €€r:   Ú_check_stagezKPipelineOptimizer._insert_sendrecv_ops_for_boundaries.<locals>._check_stageç  s¯   ø€ à!%×!4Ñ!4°RÓ!8�JØ"&×"6Ñ"6°rÓ":�KÙ%©ð ð:Ø:<¸ð?óÐ4ñ
 "Ø&¨Ò/ð ðeßekÑekØ '¨°ófóÑ/ñ %Ø&¨Ò/ð ðdßdjÑdjØ '¨°óeóÑ/ð %r<   c                 ób  •
— ‰t        | «      z   }‰t        |«      z   }||f‰‰   v ry | |z
  dkD  r/ ‰| dz
  |«        ‰| | dz
  «       ‰‰   j                  ||f«       y | |z
  dk  r/ ‰| dz   |«        ‰| | dz   «       ‰‰   j                  ||f«       y t        | |z
  «      dk(  sJ ‚‰‰   j                  ||f«       ‰j                  ‰j                  «      }‰j
                  ‰   }|| f}|dz  | z   }|‰j                  vrV‰j                  j                  |«       ‰j                  ‰j                  |<   ‰j                  }‰xj                  dz  c_        n‰j                  |   }‰j                  dk(  rÑ‰j                  ‰‰d   z   dd|i‰j                  |‰j                  |dd	d
dd|i¬«       ‰dxx   dz  cc<   t        |j                  «      }	|	d   dk  r‰j                  n|	d   |	d<   ‰j                  ‰‰d   z   dd|gid|	d|j                  ‰j                  |‰j                  |dd	d
dd|i¬«       ‰dxx   dz  cc<   y ‰j                  dk(  �rÀt        |j                  «      }	|	d   dk  r‰j                  n|	d   |	d<   t!        j"                  |	«      }
‰j$                  dkD  xr |
‰j$                  z  dk(  }d|j&                  v rŠ|j&                  j)                  d«      d   dd }‰j+                  |«      }‰j                  ‰‰d   z   dd|gid|gid|	d|j                  ‰j                  |‰j                  |dd	i¬«       ‰dxx   dz  cc<   y  ‰| |«       ‰j                  ‰‰d   z   dd|gid|gi‰j                  |‰j                  |i¬«       ‰dxx   dz  cc<   |j&                  j)                  d«      d   }‰j+                  |«      }t-        |t.        «      rd	nd}‰j                  ‰‰d   z   |r|rdndd|i‰j                  |‰j                  |ddd|d
dd‰j$                  d‰j0                  i¬«       ‰dxx   dz  cc<   d }t3        |«      t3        ‰j4                  j6                  «      k(  r‰d   }‰j4                  j8                  }n‰}‰j4                  j6                  }‰j                  |‰d   z   dd|gid|gi‰j                  |‰j                  |d|i¬«      }t3        |«      t3        ‰j4                  j:                  «      k(  r|j=                  dd «       ‰dxx   dz  cc<   ‰j                  ‰‰d   z   |r|rdnd!d|gid|	d|j                  ‰j                  |‰j                  |dd	d
dd|d‰j$                  d‰j0                  i	¬«       ‰dxx   dz  cc<   |rg|sd‰j                  ‰‰d   z   d"d|gid|gi‰j                  |‰j                  |dd	ddd#‰j$                  d$‰j0                  i¬«       ‰dxx   dz  cc<   y y y t?        d%‰j                  › d&�«      ‚)'Nr   rí   éè  zF-then-BrÀ   Úsend_v2rE   rP   TÚpeerrO   ©rÀ   r    rJ   rL   r   Úrecv_v2rF   Ú	out_shaperC   ©rÀ   r    rK   rL   Ú1F1BÚsubprogÚassign©rÀ   r    rJ   rK   rL   Úc_sync_calc_streamÚ@FÚpartial_sendÚnumÚidr   Úc_sync_comm_streamÚpipeline_flagr­   Úpartial_recvÚpartial_allgatherÚnranksÚrankz@Now only 'F-then-B' and '1F1B' are supported.The given value is r   ) r}   ru   Úabsr–   r+   r�   r1   rO   r2   Úschedule_modeÚ_insert_op_without_syncr/   r  rB   Úmicro_batch_sizerC   ÚnpÚprodÚ	mp_degreerA   rÇ   rT   r   r   Úmp_rankr•   r)   r—   rX   rœ   r±   r   )r"  r#  Úcur_devÚprev_devr™   rT   ÚpairÚpair_keyrO   Ú	var_shapeÚnumelÚuse_mpÚorigin_nameÚassociate_varÚprefix_nameÚ
prefix_varÚis_paramÚinsert_indexÚnew_op_roleÚsync_comm_opr&  Ú_insert_send_recvr[   Údevice_typeÚextra_index_inforÀ   Úinput_var_to_devicer\   r5   rÁ   s                       €€€€€€€€€€r:   rU  zPPipelineOptimizer._insert_sendrecv_ops_for_boundaries.<locals>._insert_send_recvÿ  s;  ø€ Ø)¬C°«KÑ7�GØ*¬S°«\Ñ9�HØ Ð*Ð.AÀ(Ñ.KÑKØà Ñ'¨!Ò+Ù)¨&°1©*°gÔ>Ù)¨&°&¸1±*Ô=Ø+¨HÑ5×<Ñ<Ø$ hÐ/ôð Ø 'Ñ)¨BÒ.Ù)¨&°1©*°gÔ>Ù)¨&°&¸1±*Ô=Ø+¨HÑ5×<Ñ<Ø$ hÐ/ôð ä˜v¨Ñ/Ó0°AÒ5Ð5Ð5Ø'¨Ñ1×8Ñ8¸'À8Ð9LÔMà Ÿg™g d×&7Ñ&7Ó8�GØŸ*™* XÑ.�CØ# VÐ,�Dà&¨™~°Ñ6�HØ 4×#6Ñ#6Ñ6Ø×+Ñ+×2Ñ2°4Ô8Ø6:·l±l˜×)Ñ)¨(Ñ3Ø"&§,¡,˜ØŸš¨Ñ)žà"&×"3Ñ"3°HÑ"=˜à×)Ñ)¨ZÒ7Ø×5Ñ5Ø"'Ð*:¸7Ñ*CÑ"CØ!*Ø$'¨ :à $× 3Ñ 3°XØ $× 1Ñ 1°7Ø 1°4Ø &¨Ø )¨7ð#ð	 6ô ð )¨Ó1°QÑ6Ó1Ü$(¨¯©£O˜	ð  )¨™|¨aÒ/ð !×1Ò1à!*¨1¡ð " !™ð
 ×5Ñ5Ø"'Ð*:¸7Ñ*CÑ"CØ!*Ø%*¨S¨E Nà +¨YØ '¨¯©Ø $× 3Ñ 3°WØ $× 1Ñ 1°7Ø 1°4Ø &¨Ø )¨7ð#ð	 6ô ð )¨Ó1°QÑ6Ô1Ø×+Ñ+¨vÓ5Ü$(¨¯©£O˜	ð  )¨™|¨aÒ/ð !×1Ò1à!*¨1¡ð " !™ô !#§¡¨	Ó 2˜Ø"&§.¡.°1Ñ"4ò "Ø! D§N¡NÑ2°aÑ7ð ð %¨¯©Ñ0ð +.¯(©(¯.©.¸Ó*CÀAÑ*FÀqÈÐ*L˜KØ,1¯I©I°kÓ,B˜MØ!×9Ñ9Ø&+Ð.>¸wÑ.GÑ&GØ%-Ø(+¨m¨_Ð'=Ø).°°¨à$/°Ø$+¨S¯Y©YØ$(×$7Ñ$7¸Ø$(×$5Ñ$5°wØ$5°tð'"ð :ô ð -¨WÓ5¸Ñ:Ó5Ø"á$ V¨WÔ5à×5Ñ5Ø"'Ð*:¸7Ñ*CÑ"CØ!5Ø$'¨#¨ <Ø%*¨S¨E Nà $× 3Ñ 3°XØ $× 1Ñ 1°7ð#ð 6ô 	ð )¨Ó1°QÑ6Ó1Ø&)§h¡h§n¡n°SÓ&9¸!Ñ&<˜Ø%*§Y¡Y¨{Ó%;˜
ä$.¨z¼9Ô$E™DÈ5ð !ð ×5Ñ5Ø"'Ð*:¸7Ñ*CÑ"Cá#)©Xñ "+à!/Ø$'¨ :à $× 3Ñ 3°XØ $× 1Ñ 1°7Ø 1°5Ø )¨7Ø &¨à % t§~¡~Ø $ d§l¡lð	#ð 6ô ð" )¨Ó1°QÑ6Ó1Ø'+˜Ü˜w›<¬3¨t¯}©}×/EÑ/EÓ+FÒFØ+;Ø 6ñ,˜Lð +/¯-©-×*@Ñ*@™Kà+0˜LØ*.¯-©-×*@Ñ*@˜KØ',×'DÑ'DØ".Ð1AÀ'Ñ1JÑ"JØ!5Ø$'¨#¨ <Ø%*¨S¨E Nà $× 3Ñ 3°XØ $× 1Ñ 1°;Ø )¨7ð#ð (Eó 
(˜ô ˜w›<¬3¨t¯}©}×/DÑ/DÓ+EÒEØ(×2Ñ2°?ÀBÔGØ,¨WÓ5¸Ñ:Ó5Ø×5Ñ5Ø"'Ð*:¸7Ñ*CÑ"Cá#)©Xñ "+à!/Ø%*¨S¨E Nà +¨YØ '¨¯©Ø $× 3Ñ 3°WØ $× 1Ñ 1°7Ø 1°4Ø &¨Ø )¨7à % t§~¡~Ø $ d§l¡lð#ð 6ô ð& )¨Ó1°QÑ6Ó1Ù!©(Ø!×9Ñ9Ø&+Ð.>¸wÑ.GÑ&GØ%8Ø(+¨c¨U |Ø).°°¨à$(×$7Ñ$7¸Ø$(×$5Ñ$5°wØ$5°tØ$-¨qà$,¨d¯n©nØ$*¨D¯L©Lð'"ð :ô ð -¨WÓ5¸Ñ:Ô5ð! +3˜6ô$ )ð2Ø26×2DÑ2DÐ1EÀQðHóð r<   r   )r  r  rQ   rr   r–   r/   r   r{   rT   rã   rÛ   r0   rÇ   r•   r†   )r5   r[   r   Ú
cur_devicerT   Úprev_devicerÙ   r&  rU  rV  rW  rÀ   rX  r\   rÁ   s   ``     @@@@@@@@r:   Ú#_insert_sendrecv_ops_for_boundariesz5PipelineOptimizer._insert_sendrecv_ops_for_boundaries±  sæ  ÿù€ ð !Ðà#ÐÜ"¤4¨¯	©	£?Ö3‰IˆE�2Ø×#Ñ# BÕ'Ø',Ð$Ùð 4ð
 Ø$8ñ
Ðô
 #¤4¨¯	©	£?×3‰IˆE�2ØŸ™ ×!4Ñ!4Ó5ˆJØ §¡˜~¨TÐ2Ò2ØØ×.Õ.�Ø—i‘i Ó)�à—;’;ØØ"�à×,Ñ,¨U°HÓ=�Ø�?Ø t×'=Ñ'=Ñ=Ø Ø"&×"8Ñ"8¸Ñ"B�Ká"á=D˜Ÿ™ T×%8Ñ%8Ô9È$ð  ð Ð&¨+¸D¿L¹L¸>ÈÐ9NÒ*NØà *Ò,ØàÐ#6Ñ6Ø46Ð'¨Ñ1Ø Ð,Ð0CÀHÑ0MÑMØà(×.Ñ.¨sÓ3°AÑ6¸Ñ<�õ÷0Sõ Sñj "Ü˜
×(Ñ(¨Ó-¨aÑ0Ó1Ü˜×)Ñ)¨#Ó.¨qÑ1Ó2öòY /ð	 4ðh 	×ÑÕr<   c           	      óœ  — | j                   dk(  ryt        t        t        t	        |j
                  «      «      «      «      D ]‰  \  }}| j                  |«      sŒ|j                  dk(  sJ d|j                  › �«       ‚|j                  d«      sJ ‚t        |j                  d«      «      }|| j                   z  }|j                  d|«        y y)zJ
        Scale the loss corresponding to number of micro-batches.
        r   NrÅ   z6loss_grad_op must be fill_constant op, but this op is Úvalue)r$   rÑ   Útupler  r  rQ   rš   r    rö   Úfloatr–   r±   )r5   r[   rÀ   r\   Ú
loss_scales        r:   Ú_insert_loss_scalez$PipelineOptimizer._insert_loss_scaleÚ  s½   € ð ×!Ñ! QÒ&ØÜ!¤%¬	´$°u·y±y³/Ó(BÓ"CÖD‰IˆE�2Ø×$Ñ$ RÕ(Ø—w‘w /Ò1ð ð&Ø&(§g¡g Yð0óÐ1ð —{‘{ 7Ô+Ð+Ð+Ü" 2§7¡7¨7Ó#3Ó4�
Ø'¨$×*@Ñ*@Ñ@�
Ø—‘˜W jÔ1Ùñ Er<   c                 óŒ  — t        |j                  «      D ]¬  \  }}| j                  |«      sŒ|j                  }|j                  }||z   }|j
                  dk(  s|j
                  dk(  rŒT|D ]T  }t        j                  «       |vrŒ|j                  t        j                  «       «      }|dz   }	| j                  |||	«       ŒV Œ® y )NrD   r8  ú@MERGED)
r  rQ   rr   r{   rS   r    r   rï   Ústriprá   )
r5   r[   rÀ   r\   r©   Úoutput_namesÚin_out_namesrA   rÂ   Únew_grad_names
             r:   Ú_rename_gradient_var_namez+PipelineOptimizer._rename_gradient_var_nameì  s¸   € Ü" 5§9¡9Ö-‰IˆE�2Ø×'Ñ'¨Ô+ØØ×,Ñ,ˆKØ×.Ñ.ˆLØ&¨Ñ5ˆLØ�w‰w˜&Ò  B§G¡GÐ/CÒ$CØó %�Ü×'Ñ'Ó)°Ñ5ØØ!ŸZ™Z¬×(<Ñ(<Ó(>Ó?�
Ø $ yÑ 0�Ø× Ñ   T¨=Õ9ñ %ñ .r<   c                 ó
  — |r|j                   nd}|r,|j                  r | j                  |||j                  |«      }|S g }d}|rdnd}	|rt        j
                  nd}
t        t        t        t        |j                  «      «      «      «      D �]/  \  }}| j                  |«      rs|j                  dk(  rd|j                  d   }|j                  d   }|j                  d«      | j                   v r)|j#                  dd	«      |k(  sJ ‚|j%                  |«       Œ‹| j'                  |«      r|€|d
z   }| j'                  |«      sŒµ| j(                  |j*                  v sŒÎ|j-                  | j(                  «      }t/        |«      dk(  rŒøt/        |«      dz  dk(  sJ ‚t1        dt/        |«      d«      D �]  }d}||   }|j3                  |«      sŒd|v rŒ"|t5        j6                  «       z   }||	z   }|j3                  |«      s!| j9                  ||j:                  |   ||
«       |j3                  |«      sJ ‚|j=                  |«      }|j=                  |«      }d|_        |jA                  ||z   di d|gid|jB                  d|jD                  dtG        d«      | jH                  | jJ                  jL                  jN                  i¬«       |d
z  }||d
z      }|j:                  |   }d|v }||u}|r||dz   }| j9                  ||||
«      }d|_        |jA                  ||z   dd|id|id|jD                  d|jD                  | jH                  | jJ                  jP                  i¬«       |d
z  }|}|jA                  ||z   dd||gid|i| jH                  | jJ                  jP                  i¬«       |d
z  }|jS                  |«       �Œ �Œ2 |s|S d}t        t        t        t        |j                  «      «      «      «      D ]   \  }}| j'                  |«      sŒ|�Œ|d
z   } n |€J ‚|D ]å  }|j#                  dd	«      }|j#                  dd	«      }|j3                  |«      s | j9                  ||j:                  |   |«       |j3                  |«      sJ ‚|j=                  |«      }|j=                  |«      }d|_        |jA                  |dd|id|id|jD                  d|jD                  | jH                  | jJ                  jL                  i¬«       Œç |S )zz
        Create a new merged gradient for each parameter and accumulate the
        corresponding gradient to it.
        FNú@MERGED@FP16rc  rD   r   ú@GRADrÏ   r­   r   rü   ú
@BroadCastTrÅ   rF   rB   rC   r]  r2  Ú	cast_fp16ú@TMPrE   rG   rH   rg   z@FP16z@GRAD@MERGED@FP16)*Úfp16_allreduceÚfuse_grad_mergeÚ_accumulate_gradients_with_fuseÚfuse_grad_size_in_MBr   Úfloat16rÑ   r^  r  r  rQ   rr   r    r{   rS   rd  r0   rÐ   ry   r    r-   r”   r–   rx   r  Úhas_varr   rï   rç   r�   rT   ri   rW   rB   rC   r_  r+   r)   rX   r  r—   ru   )r5   r[   Úpp_allreduce_in_optimizeÚstrategyÚshardro  Úfused_gradient_namesÚmerged_gradient_namesÚfirst_opt_op_idxÚmerged_suffixrC   rÀ   r\   Úin_namer]   r  r	  r_   rÂ   Úparam_grad_nameÚmerged_param_grad_nameÚparam_grad_varÚmerged_param_grad_varr
  Úgrad_varÚis_fp16_gradÚ	need_castÚcast_grad_var_nameÚcast_grad_varÚfp16_grad_nameÚfp16_grad_vars                                  r:   Ú_accumulate_gradientsz'PipelineOptimizer._accumulate_gradientsþ  sT  € ñ 5=˜×0Ò0À%ˆÙ˜×0Ò0Ø#'×#GÑ#GØ�~ x×'DÑ'DÀeó$Ð ð (Ð'à "ÐØÐá*8™¸iˆÙ"0”—’°dˆä!¤%¬	´$°u·y±y³/Ó(BÓ"C×D‰IˆE�2à×#Ñ# BÔ'¨B¯G©G°vÒ,=Ø×,Ñ,¨QÑ/�Ø×.Ñ.¨qÑ1�Ø—>‘> 'Ó*¨d×.DÑ.DÑDØ"Ÿ?™?¨<¸Ó<ÀÒHÐHÐHØ×$Ñ$ UÔ+Øà×#Ñ# BÔ'Ð,<Ð,DØ#(¨1¡9Ð ð ×#Ñ# BÕ'Ø×%Ñ%¨¯©Ò6à Ÿg™g d×&;Ñ&;Ó<�Ü�{Ó# qÒ(ØÜ˜;Ó'¨!Ñ+¨qÒ0Ð0Ð0Ü˜q¤# kÓ"2°A×6�AØ�FØ!,¨Q¡�JØ Ÿ=™=¨Ô4Ø Ø# zÑ1Ø à&0´4×3GÑ3GÓ3IÑ&I�OØ-<¸}Ñ-LÐ*Ø Ÿ=™=Ð)?Ô@Ø×(Ñ(Ø!Ø!ŸJ™J zÑ2Ø2Ø!ô	ð !Ÿ=™=Ð)?Ô@Ð@Ð@à%*§Y¡Y¨Ó%?�NØ,1¯I©IÐ6LÓ,MÐ)Ø8<Ð)Ô5Ø×$Ñ$Ø.°Ñ7Ø,Ø!Ø!&Ð)>Ð(?Ð @à#Ð%:×%@Ñ%@Ø#Ð%:×%@Ñ%@Ø#¤U¨1£Xà ×-Ñ-¨t¯}©}×/EÑ/E×/MÑ/Mðð %ô ð ˜a‘K�FØ +¨A°©EÑ 2�IØ$Ÿz™z¨)Ñ4�Hà#.°)Ð#;�LØ ,°NÐ B�Iá ð
 .=¸vÑ-EÐ*Ø(,×(8Ñ(8Ø! >Ð3EÀuó)˜ð 5:˜Ô1Ø×(Ñ(Ø"2°VÑ";Ø!'Ø$'¨ ?Ø%*¨MÐ$:à *¨H¯N©NØ +¨]×-@Ñ-@Ø $× 1Ñ 1°4·=±=×3IÑ3Ið#ð )ô 
ð  !™˜Ø#0˜à×$Ñ$Ø.°Ñ7Ø"Ø #Ð&;¸XÐ%FÐGØ!&Ð(=Ð >à ×-Ñ-¨t¯}©}×/EÑ/Eðð %ô ð ˜a‘K�FØ)×0Ñ0Ð1GÖHòY 7ð- EñH Ø(Ð(àÐÜ!¤%¬	´$°u·y±y³/Ó(BÓ"CÖD‰IˆE�2Ø×#Ñ# BÕ'Ð,<Ñ,DØ#(¨1¡9Ð Ùð Eð  Ð+Ð+Ð+ó
 4ˆNØ&×.Ñ.¨w¸Ó;ˆIØ'×/Ñ/Ð0CÀRÓHˆJà—=‘= Ô+Ø× Ñ  ¨¯
©
°:Ñ(>À	ÔJØ—=‘= Ô+Ð+Ð+à!ŸI™I nÓ5ˆMØ—y‘y Ó+ˆHØ#(ˆHÔ à×ÑØ&ØØ˜]Ð+Ø Ð)à × 3Ñ 3Ø §¡Ø×%Ñ% t§}¡}×'=Ñ'=ðð õ 
ð 4ð0 %Ð$r<   c                 óÊ  — | j                  ||«      }g }|rdnd}|rt        j                  nt        j                  }d}	d }
|D �]#  \  }}|j	                  |«      }|j                  |t        j                  «       z   |z   ||j                  dd¬«      }|j	                  |«      }t        |d«      r|j                  |_
        | j                  |«      }t        |«      dk(  s|	|z   |kD  s|j                  |
k7  r&|j                  |g|g|gf«       |j                  }
d}	ŒÚ|d	   d   j                  |«       |d	   d
   j                  |«       |d	   d   j                  |«       |	|z  }	�Œ& g }g }|D ]»  }|d   }|d   }|j                  d|d   j                  › �|d   j                  dd¬«      }|d   j                  t        j                  k(  rdnd}|d|d   j                  › �z   }|j                  ||d   j                  dd¬«      }|j                  |«       |j                  |«       Œ½ t        |«      t        |«      k(  sJ ‚t        |«      t        |«      k(  sJ ‚d }t!        |j"                  «      D ]  \  }}| j%                  |«      sŒ|�Œ|} n |€J ‚d}t'        t        |«      «      D ]ù  }||   }||   }||   d   }||   d
   } ||   d   }!|j)                  ||z   dd| i||dœddddddd|d   j                  | j*                  | j,                  j.                  dt        j0                  d«      ddi¬«       |d
z  }|j)                  ||z   dd| i|!|dœddddddddddd|!d   j                  | j*                  | j,                  j2                  j4                  i¬«       |d
z  }Œû ||z  }d}t'        t        |«      «      D ]ä  }||   }||   }d|j                  v }"|"|u}#|#r€|j                  dz   }$|j                  |$|dd¬«      }%|j7                  ||z   dd|id |%id!|j                  d"|%j                  | j*                  | j,                  j.                  i¬#«       |d
z  }|%}|j7                  ||z   d$d||gid |i| j*                  | j,                  j.                  i¬#«       |d
z  }Œæ |�r|D ]ÿ  \  }}|j	                  |«      }|t        j                  «       z   dz   }&|j9                  |&«      sJ ‚|j	                  |&«      }'|t        j                  «       z   dz   }(|j                  |(t        j                  |j                  dd¬«      })|j7                  ||z   dd|'id |)id!t        j                  d"t        j                  | j*                  | j,                  j2                  i¬#«       |d
z  }�Œ t'        t        |«      «      D ]  }||   j                  ||<   Œ ||fS )%Nrj  rc  g        TF)rA   rC   rB   ri   rk   ré   r   rí   r   rü   Ú
FusedGrad_)rA   rC   ri   rk   zFusedMergedGrad.cast_fp16.ÚFusedMergedGradÚ_Úcoalesce_tensorÚInput)ÚOutputÚFusedOutputÚuser_defined_size_of_dtypeÚ	copy_dataÚ	use_alignrC   Úset_constantÚnpuÚconstantrI   rm  rn  rD   rE   rF   rG   rH   r2  rg   )Ú_sort_grad_param_by_dtyper   rs  Úfloat32rT   rV   r   rï   rB   r#   ré   Ú_get_var_sizerx   rC   ru   rA   r  rQ   r    r  r@  r+   r)   r—   Úis_compiled_with_custom_devicerX   r  rW   rt  )*r5   Ú
main_blockÚfp16Ú
fused_sizeÚgrad_param_pairsrz  Úgrad_param_segmentsr{  rC   Úcur_sizeÚ
last_dtypeÚgradÚparamÚ	real_gradÚmerged_grad_varÚ
real_paramÚtmp_sizeÚfused_gradientsÚfused_merged_gradientsÚgrad_param_segmentÚgrad_segmentÚmerged_grad_segmentÚ
fused_gradÚfused_merged_grad_name_prefixÚfused_merged_grad_nameÚfused_merged_gradÚfirst_back_op_idxrÀ   r\   r_   r	  ÚgradsÚparamsÚmerged_gradsr‚  rƒ  r„  r…  r†  Ú	fp16_gradÚfp32_grad_nameÚ	fp32_grads*                                             r:   Ú&_insert_accumulate_gradients_with_fusez8PipelineOptimizer._insert_accumulate_gradients_with_fuse�  s  € ð  ×9Ñ9ØÐ(ó
Ðð !ÐÙ*.™°IˆÙ"&”—’¬F¯N©NˆØˆØˆ
ä+‰KˆD�%Ø"Ÿ™ tÓ,ˆIà(×3Ñ3ØœT×1Ñ1Ó3Ñ3°mÑCØØ—o‘oØ Ø#ð 4ó ˆOð $Ÿ™¨Ó.ˆJÜ�zÐ#3Ô4Ø1;×1JÑ1J�Ô.Ø×)Ñ)¨)Ó4ˆHô
 Ð'Ó(¨AÒ-Ø˜hÑ&¨Ò3Ø—?‘? jÒ0à#×*Ñ*Ø�[ : ,°Ð0AÐBôð 'Ÿ_™_�
Ø‘à# BÑ'¨Ñ*×1Ñ1°)Ô<Ø# BÑ'¨Ñ*×1Ñ1°*Ô=Ø# BÑ'¨Ñ*×1Ñ1°/ÔBØ˜HÑ$’ð? ,ðB ˆØ!#Ðã"5ÐØ-¨aÑ0ˆLØ"4°QÑ"7ÐØ#×.Ñ.Ø! ,¨q¡/×"6Ñ"6Ð!7Ð8Ø" 1‘o×+Ñ+Ø!Ø#ð	 /ó ˆJð ' qÑ)×/Ñ/´6·>±>ÒAñ -à&ð *ð .ØÐ)¨!Ñ,×1Ñ1Ð2Ð3ñ4ð #ð !+× 5Ñ 5Ø+Ø)¨!Ñ,×2Ñ2Ø Ø#ð	 !6ó !Ðð ×"Ñ" :Ô.Ø"×)Ñ)Ð*;Õ<ð5 #6ô8 �?Ó#¤sÐ+>Ó'?Ò?Ð?Ð?ÜÐ)Ó*¬cÐ2EÓ.FÒFÐFÐFð !ÐÜ" :§>¡>Ö2‰IˆE�2Ø×#Ñ# BÕ'Ð,=Ñ,EØ$)Ð!Ùð 3ð !Ð,Ð,Ð,ØˆÜ”sÐ.Ó/Ö0ˆAØ(¨Ñ+ˆJØ 6°qÑ 9ÐØ'¨Ñ*¨1Ñ-ˆEØ(¨Ñ+¨AÑ.ˆFØ.¨qÑ1°!Ñ4ˆLØ×.Ñ.Ø! FÑ*Ø&Ø Ð(Ø#(¸ÑDð 1°!Ø Ø Ø˜U 1™XŸ^™^Ø×%Ñ% t§}¡}×'=Ñ'=ð
 #¤D×$GÑ$GÈÓ$NØ ð-ð /ô ð< �a‰KˆFð ×.Ñ.Ø! FÑ*Ø&Ø Ð(à*Ø#4ñð
 1°!Ø" DØ Ø Ø Ø˜\¨!™_×2Ñ2Ø×%Ñ% t§}¡}×'=Ñ'=×'EÑ'Eðð /ô ð$ �a‰K‰Fðu 1ðz 	˜FÑ"ÐØˆÜ”s˜?Ó+Ö,ˆAØ(¨Ñ+ˆJØ 6°qÑ 9ÐØ&¨*¯/©/Ð9ˆLØ$¨DÐ0ˆIÙð &0§_¡_°vÑ%=Ð"Ø *× 5Ñ 5Ø+ØØ %Ø"'ð	 !6ó !�ð ×%Ñ%Ø*¨VÑ3ØØ Ð,Ø" MÐ2à" J×$4Ñ$4Ø# ]×%8Ñ%8Ø×)Ñ)¨4¯=©=×+AÑ+Aðð &ô 
ð ˜!‘�Ø*�
Ø×!Ñ!Ø&¨Ñ/ØØÐ/°Ð<Ð=ØÐ 1Ð2Ø×(Ñ(¨$¯-©-×*@Ñ*@ÐAð "ô ð �a‰K‰FðG -òJ ã/‘��eØ&ŸN™N¨4Ó0�	Ø!&¬×)=Ñ)=Ó)?Ñ!?À.Ñ!P�Ø!×)Ñ)¨.Ô9Ð9Ð9Ø&ŸN™N¨>Ó:�	Ø!&¬×)=Ñ)=Ó)?Ñ!?À)Ñ!K�Ø&×1Ñ1Ø'Ü Ÿ.™.Ø#Ÿ/™/Ø %Ø"'ð 2ó �	ð ×%Ñ%Ø*¨VÑ3ØØ Ð+Ø" IÐ.à"¤F§N¡NØ#¤V§^¡^Ø×)Ñ)¨4¯=©=×+AÑ+Aðð &ô 
ð ˜!‘’ð1  0ô6 ”sÐ1Ó2Ö3ˆAØ(>¸qÑ(A×(FÑ(FÐ" 1Ò%ð 4ð &Ð'7Ð7Ð7r<   c           	      óÐ  — d }g }t        t        t        t        |j                  «      «      «      «      D �]x  \  }}| j                  |«      rs|j                  dk(  rd|j                  d   }	|j                  d   }
|
j                  d«      | j                  v r)|	j                  dd«      |
k(  sJ ‚|j                  |«       Œ‹| j                  |«      r!|€|dz   }|t        |j                  «      k(  r y | j                  |«      sŒÏ| j                  |j                   v sŒè|j#                  | j                  «      }t        |«      dk(  r�Œt        |«      dz  dk(  sJ ‚t%        dt        |«      d«      D ]:  }||   }|j'                  |«      sŒd|v rŒ|j)                  ||dz      ||   f«       Œ< �Œ{ t        |«      dk(  ry |r|j*                  nd}t%        |«      D �cg c]  }g ‘Œ }}|D ]>  }|r|j-                  |d   «      nd}d|cxk  r|k  sJ ‚ J ‚||   j)                  |«       Œ@ g }|D ]  }| j/                  |||||«      \  }}||z  }Œ! |j1                  «        |S c c}w )	NrD   r   rk  rÏ   r­   r   rü   rl  )rÑ   r^  r  r  rQ   rr   r    r{   rS   rd  r0   rÐ   ry   r    rx   r-   r”   r–   r  rt  ru   Ú
worker_numrµ   r¸  r†   )r5   r›  rœ  r�  rw  rz  rž  rÀ   r\   r|  r]   r  r	  rÂ   r<  rŒ  Údevice_to_pairsrH  Úroot_idÚall_fused_merged_gradientsÚpairsr©  s                         r:   rq  z1PipelineOptimizer._accumulate_gradients_with_fuse|  s†  € ð  ÐØÐä!¤%¬	´$°z·~±~Ó2FÓ(GÓ"H×I‰IˆE�2à×#Ñ# BÔ'¨B¯G©G°vÒ,=Ø×,Ñ,¨QÑ/�Ø×.Ñ.¨qÑ1�Ø—>‘> 'Ó*¨d×.DÑ.DÑDØ"Ÿ?™?¨<¸Ó<ÀÒHÐHÐHØ×)Ñ)¨%Ô0Øà×#Ñ# BÔ'Ð,<Ð,DØ#(¨1¡9Ð à#¤s¨:¯>©>Ó':Ò:Ùà×#Ñ# BÕ'Ø×%Ñ%¨¯©Ò6à Ÿg™g d×&;Ñ&;Ó<�Ü�{Ó# qÒ(ÙÜ˜;Ó'¨!Ñ+¨qÒ0Ð0Ð0Ü˜q¤# kÓ"2°AÖ6�AØ!,¨Q¡�JØ%×-Ñ-¨jÔ9Ø Ø# zÑ1Ø Ø$×+Ñ+Ø$ Q¨¡UÑ+¨[¸©^Ð<õò 7ð/ JôB ÐÓ  AÒ%Øá%*�×!Ò!°ˆÜ',¨V¤}Ó5¡} !š2 }ˆÐ5Û$ˆDÙ/4�e—l‘l 4¨¡7Ô+¸!ˆGØ˜Ô( &Ò(Ð(Ñ(Ð(Ð(Ø˜GÑ$×+Ñ+¨DÕ1ð %ð
 &(Ð"Û$ˆEð ×;Ñ;Ø˜D *¨eÐ5EóñØ&Ø ð 'Ð*@Ñ@Ñ&ð %ð 	×!Ñ!Ô#Ø)Ð)ùò# 6s   Ç	I#c                 óZ  — g }g }g }|D ]{  }|j                  |d   «      j                  }|t        j                  k(  r|j	                  |«       ŒF|t        j
                  k(  r|j	                  |«       Œk|j	                  |«       Œ} |}|j                  |«       |j                  |«       |S )Nr   )rT   rC   r   r˜  ru   rs  Úextend)	r5   r›  rž  Ú
fp16_pairsÚ
fp32_pairsÚother_pairsr¾  rC   Úsorted_pairss	            r:   r—  z+PipelineOptimizer._sort_grad_param_by_dtypeº  sž   € àˆ
Øˆ
ØˆÛ%ˆEØ—N‘N 5¨¡8Ó,×2Ñ2ˆEØœŸ™Ò&Ø×!Ñ! %Õ(Øœ&Ÿ.™.Ò(Ø×!Ñ! %Õ(à×"Ñ" 5Õ)ð &ð "ˆØ×Ñ˜JÔ'Ø×Ñ˜KÔ(ØÐr<   c                 ó
  — t         j                  j                  j                  dt         j                  j                  j                  dt         j                  j                  j
                  dt         j                  j                  j                  dt         j                  j                  j                  dt         j                  j                  j                  dt         j                  j                  j                  dt         j                  j                  j                  dt         j                  j                  j                  di	}d|j                  vsJ ‚t        d„ |j                  d«      ||j                     z  dz  dz  S )Nrü   é   é   r   rí   c                 ó   — | |z  S r“   © )ÚxÚys     r:   Ú<lambda>z1PipelineOptimizer._get_var_size.<locals>.<lambda>Ú  s   €   A¢r<   g      �@)r   r   r€   ÚFP16ÚBF16ÚFP32ÚFP64ÚINT16ÚINT32ÚINT64ÚBOOLÚUINT8rB   r   rC   )r5   rT   Údtype_to_sizes      r:   r™  zPipelineOptimizer._get_var_sizeÌ  s  € ä�L‰L× Ñ ×%Ñ% qÜ�L‰L× Ñ ×%Ñ% qÜ�L‰L× Ñ ×%Ñ% qÜ�L‰L× Ñ ×%Ñ% qÜ�L‰L× Ñ ×&Ñ&¨Ü�L‰L× Ñ ×&Ñ&¨Ü�L‰L× Ñ ×&Ñ&¨Ü�L‰L× Ñ ×%Ñ% qÜ�L‰L× Ñ ×&Ñ&¨ð

ˆð ˜Ÿ™Ñ"Ð"Ð"äÑ% s§y¡y°!Ó4Ø˜CŸI™IÑ&ñ'àñð ñð	
r<   c                 ó  — |j                   }|D ]î  }|j                  d«      j                  D ]Î  }|j                  d«      sŒ|j	                  d«      j
                  }|j                  |«      }|j                  d¬«      }|j                  D ]9  }	|	j                  }
|j                  j                  «       }|j                  |
«       Œ; |j                  «        | j                  ||«       |j                  d|«       ŒÐ Œð y )Nr   Ú	sub_block)Ú
parent_idx)r¶   r[   rQ   rö   r–   r7  Ú_create_blockrR   r¯   r°   r†   r‘   r±   )r5   r›  r¹   r²   Úprogr\   Úorigin_sub_block_idÚorigin_sub_blockÚnew_sub_blockÚsub_opr·   r¸   s               r:   Ú_add_sub_blocksz!PipelineOptimizer._add_sub_blocksà  sÞ   € Ø!×)Ñ)ˆÛ ˆDØ—j‘j “m×'Ô'�Ø—{‘{ ;Ô/ØØ&(§g¡g¨kÓ&:×&=Ñ&=Ð#Ø#/×#5Ñ#5Ð6IÓ#JÐ Ø $× 2Ñ 2¸aÐ 2Ó @�Ø.×2Ô2�FØ$Ÿk™k�GØ)×.Ñ.×8Ñ8Ó:�EØ—O‘O GÕ,ð 3ð ×,Ñ,Ô.Ø×!Ñ! -Ð1AÔBØ—‘˜[¨-Õ8ñ (ñ !r<   c                 óš   — |j                   D ]<  }|j                  |j                  «      sŒ|j                  | j                  «      }|c S  y r“   )rQ   r  r    r–   r/   )r5   r[   r\   rù   s       r:   Ú_get_device_infoz"PipelineOptimizer._get_device_infoñ  s=   € Ø—)”)ˆBØ—>‘> "§'¡'Ô*ØØŸ™ × 3Ñ 3Ó4ˆIØÒñ	 r<   c                 óf  — i }|D ]m  }|j                  d«      }|j                  D ]K  }|dk(  rŒ	|j                  |«      }|j                  sŒ'||vrg ||<   |||   vsŒ8||   j	                  |«       ŒM Œo t        |j                  «       «      D ]%  }t        ||   «      dk(  sŒ|j                  |«       Œ' i }	|j                  «       D ]æ  }||   D ]Ü  }|j                  d«      }|j                  D ]º  }
|
j                  dk(  s-|
j                  dk(  s|
j                  dk(  s|
j                  dk(  rŒ@|
j                  | j                  «      t        | j                  j                  j                   «      k(  rŒ†||
j"                  j%                  «       v sŒ£||	vsJ d|› d	|
› d
�«       ‚||	|<    ŒÜ ŒÞ Œè |j                  «       D �]m  }||	vrŒ	|	|   }|j                  d«      }| j'                  |«      }t        |j)                  d«      d   «      }||   }|D �]  }||k(  rŒ
|j                  d«      }| j'                  |«      }t        |j)                  d«      d   «      }||f}|dz  |z   }|| j*                  vrV| j*                  j	                  |«       | j,                  | j.                  |<   | j,                  }| xj,                  dz  c_        n| j.                  |   }|j1                  ddd|j                  |«      i| j2                  |dd| j                  | j                  j                   d|d|i¬«       |j1                  ddd|j                  |«      gid|j                  |«      j4                  d|j                  |«      j6                  | j2                  |dd| j                  | j                  j                   d|d|i¬«       |j1                  ddd|j                  |«      gid|j                  |«      gi| j2                  || j                  | j                  j                   d|i¬«       �Œ �Œp y)zu
        Special Case: process persistable vars that exist in
        multiple sections, e.g., shared weight
        r   Údouble_buffer_0r   r,  r  r  re   z two sections write the same var(z): second op r   rÆ   r(  r)  rE   rP   Fr*  rO   r+  rF   r-  rC   r.  r8  r2  N)r[   r�   rT   ri   ru   r  Úkeysrx   ÚpoprQ   r    r–   r+   r•   r)   rX   r  rR   rS   râ  rÇ   r1   rO   r2   rW   r/   rB   rC   )r5   r²   Ústartup_progr¹   Úvar_inforÛ  r[   rÁ   rT   Ú
write_infor\   Ú
write_progÚwrite_blockÚwrite_deviceÚwrite_dev_indexÚ	all_progsÚ
read_blockÚread_deviceÚread_dev_indexrH  rI  rO   s                         r:   Ú+_process_persistable_vars_in_multi_sectionsz=PipelineOptimizer._process_persistable_vars_in_multi_sectionsø  sG  € ð ˆÛ ˆDØ—J‘J˜q“MˆEØ!ŸJœJ�ØÐ0Ò0ØØ—i‘i Ó)�Ø—’ØØ 8Ñ+Ø)+�H˜XÑ&Ø˜x¨Ñ1Ò1Ø˜XÑ&×-Ñ-¨dÕ3ñ 'ð !ô ˜XŸ]™]›_Ö-ˆHÜ�8˜HÑ%Ó&¨!Ó+Ø—‘˜XÕ&ð .ð ˆ
Ø Ÿ™žˆHØ  Ô*�ØŸ
™
 1›�ØŸ)œ)�BàŸ™ 9Ò,ØŸ7™7Ð&8Ò8ØŸ7™7 fÒ,ØŸ7™7Ð&;Ò;à à—w‘w˜t×0Ñ0Ó1´SØŸ™×.Ñ.×6Ñ6ó6ò ð !Ø 2§7¡7×#;Ñ#;Ó#=Ò=Ø'¨zÑ9ð Ø>¸x¸jð I"Ø"$  Qð(óÐ9ð 04˜
 8Ñ,Ùñ' $ñ +ð (ð0 !Ÿ™ŸˆHà˜zÑ)Øð $ HÑ-ˆJØ$×*Ñ*¨1Ó-ˆKØ×0Ñ0°Ó=ˆLÜ! ,×"4Ñ"4°SÓ"9¸!Ñ"<Ó=ˆOØ  Ñ*ˆIÜ!�Ø˜:Ò%ØØ!ŸZ™Z¨›]�
Ø"×3Ñ3°JÓ?�Ü!$ [×%6Ñ%6°sÓ%;¸AÑ%>Ó!?�Ø'¨Ð8�Ø*¨TÑ1°NÑB�Ø˜t×2Ñ2Ñ2Ø×'Ñ'×.Ñ.¨tÔ4Ø26·,±,�D×%Ñ% hÑ/Ø"Ÿl™l�GØ—L’L AÑ%–Là"×/Ñ/°Ñ9�Gà×&Ñ&ØØ"à˜[Ÿ_™_¨XÓ6ðð ×+Ñ+¨\Ø)¨5ð ×)Ñ)¨4¯=©=×+@Ñ+@Ø Ø! 7ðð 'ô ð  ×%Ñ%ØØ"Ø" Z§^¡^°HÓ%=Ð$>Ð?à# Z§^¡^°HÓ%=×%CÑ%CØ §¡°Ó!9×!?Ñ!?Ø×+Ñ+¨[Ø)¨5ð ×)Ñ)¨4¯=©=×+@Ñ+@Ø Ø! 7ð
ð	 &ô ð  ×%Ñ%ØØ-Ø *§.¡.°Ó":Ð!;Ð<Ø" Z§^¡^°HÓ%=Ð$>Ð?à×+Ñ+¨[ð ×)Ñ)¨4¯=©=×+@Ñ+@Ø! 7ðð &ö òa "ñ (r<   c                 ó�   — |j                   j                  d«      xr* |j                   j                  d«      j                  d«      S )NÚop_namescopez/gradient_clip©rR   rö   r–   Ú
startswithr�   s     r:   rz   z&PipelineOptimizer._is_gradient_clip_opv  s;   € Ø�w‰w×Ñ Ó/ò '°B·G±G·L±LØó5
ç
‰*Ð%Ó
&ð	'r<   c                 ó�   — |j                   j                  d«      xr* |j                   j                  d«      j                  d«      S )Nrô  z/regularizationrõ  r�   s     r:   r  z'PipelineOptimizer._is_regularization_op{  s;   € Ø�w‰w×Ñ Ó/ò (°B·G±G·L±LØó5
ç
‰*Ð&Ó
'ð	(r<   c                 óv   — |j                   j                  d«      xr d|j                   j                  d«      v S )Nrô  zweight decay)rR   rö   r–   r�   s     r:   Ú_is_weight_decay_opz%PipelineOptimizer._is_weight_decay_op€  s6   € à�w‰w×ÑØó
ò =à §¡§¡¨^Ó <Ð<ð	=r<   c                 ó  — t        t        «      }t        t        «      }t        |j                  «      D ]S  \  }}|j                  D ]  }||   j                  ||g«       Œ |j                  D ]  }||   j                  ||g«       Œ ŒU ||fS )z2
        Get info of op input and output.
        )r   r  r  rQ   r{   ru   rS   )r5   r[   r3   r4   rÀ   r\   rÁ   s          r:   Ú_get_input_output_infoz(PipelineOptimizer._get_input_output_info†  s‹   € ô
 '¤tÓ,Ðä%¤dÓ+ˆä" 5§9¡9Ö-‰IˆE�2Ø×.Ô.�Ø Ñ)×0Ñ0°"°e°Õ=ð /à×/Ô/�Ø  Ñ*×1Ñ1°2°u°+Õ>ñ 0ð .ð   Ð0Ð0r<   c           	      óª  — | j                   dk7  ry|j                  d«      }| j                  dk(  rdnd}d}t        |j                  «      D ]*  \  }}|j
                  |k(  sŒ| j                  |«      sŒ(|} n |€yd}t        t        |j                  «      «      D ]¥  \  }}||k\  r n›|j
                  dk(  sŒ|j                  d«      sŒ/|j                  d   }|j                  |«      }	|j                  ||z   d	¬
«       |dz  }|j                  |dd|	gid|	gi| j                  | j                  j                  i¬«       Œ§ |j!                  «        y)zC
        optimize forward send's sync_comm_stream schedule
        r/  Nr   r   r,  r:  r8  r9  F)ÚsyncÚnoprE   rF   r2  )r?  r[   rD  r  rQ   r    r    r  rö   r{   rT   ry   r@  r+   r)   r—   r†   )
r5   r¶   r[   Ú	recv_typeÚbackward_recv_indexrÀ   r\   r_   rÁ   rT   s
             r:   Ú_optimize_forward_send_syncz-PipelineOptimizer._optimize_forward_send_sync—  sT  € ð ×Ñ Ò'Øà—‘˜aÓ ˆà!%§¡°1Ò!4‘I¸.ˆ	Ø"ÐÜ" 5§9¡9Ö-‰IˆE�2Ø�w‰w˜)Ó#¨×(<Ñ(<¸RÕ(@Ø&+Ð#Ùð .ð Ð&ØàˆÜ"¤4¨¯	©	£?Ö3‰IˆE�2ØÐ+Ò+ÙØ�w‰wÐ.Ó.°2·;±;¸Õ3OØ×-Ñ-¨aÑ0�Ø—i‘i Ó)�Ø× Ñ  ¨¡°eÐ Ô<Ø˜!‘�ð ×-Ñ-Ø-ØØ # ˜<Ø" S E˜NØ×,Ñ,¨d¯m©m×.DÑ.DÐEð .õ ð 4ð* 	×ÑÕr<   c           	      ó¤  — d}d}|j                  «       }t        |j                  «       j                  «      }t        |«      D �]ú  }d}|j                  «       j                  |   }t	        |j                  | j                  «      «      }	|	t	        | j                  j                  «      k(  r|€|}|j                  dk7  r.|j                  dk7  r|j                  dk7  r|j                  dk7  rŒª|	t	        | j                  j                  «      k(  r||k(  r|dz  }Œ×|}n?|	t	        | j                  j                  «      k(  r||k(  r|dz  }�Œ|}nt        d|	› �«      ‚i }
|j                  D ]  }|j                  |«      |
|<   Œ i }|j                  D ]  }|j                  |«      ||<   Œ |j!                  ||j                  |
||j#                  «       ¬	«       |j%                  |dz   «       |	t	        | j                  j                  «      k(  r|dz  }�ŒÒ|	t	        | j                  j                  «      k(  s�Œö|dz  }�Œý |j'                  «        y)
zc
        A pass to move the recv op to the beginning of
        the forward/backward phase
        r   Nr:  r;  rþ  r,  r   zUnknown op_role: r2  )r®   rx   rQ   r  r•   r–   r+   r)   r—   r    rœ   r   r©   rs   re  r  r@  Ú	all_attrsry   r†   )r5   r¶   Úforward_insert_indexÚbackward_insert_indexr[   Únum_opsr	  rR  r\   r™   Ú	op_inputsrA   Ú
op_outputss                r:   Ú_mv_head_recvzPipelineOptimizer._mv_head_recvÃ  s)  € ð
  !ÐØ $ÐØ×$Ñ$Ó&ˆÜ�g×*Ñ*Ó,×0Ñ0Ó1ˆÜ�w—ˆAØˆLØ×%Ñ%Ó'×+Ñ+¨AÑ.ˆBÜ˜"Ÿ'™' $×"3Ñ"3Ó4Ó5ˆGàœ3˜tŸ}™}×5Ñ5Ó6Ò6Ø)Ð1à()Ð%à—‘˜>Ò)Ø—G‘GÐ2Ò2Ø—G‘G˜uÒ$Ø—G‘G˜yÒ(àØœ#˜dŸm™m×3Ñ3Ó4Ò4ØÐ,Ò,Ø(¨AÑ-Ð(ØØ3‘ØœC §¡× 6Ñ 6Ó7Ò7ØÐ-Ò-Ø)¨QÑ.Ð)ÙØ4‘ä Ð#4°W°IÐ!>Ó?Ð?ØˆIØŸœ�Ø"$§(¡(¨4£.�	˜$’ð 'àˆJØŸœ�Ø#%§9¡9¨T£?�
˜4Ò ð (à×)Ñ)Ø"Ø—W‘WØ Ø"Ø—l‘l“nð *ô ð ×Ñ˜Q ™UÔ#Øœ#˜dŸm™m×3Ñ3Ó4Ò4Ø$¨Ñ)Ò$ØœC §¡× 6Ñ 6Ó7Ô7Ø%¨Ñ*Ò%ð[  ð\ 	×ÑÕr<   c                 óÎ  — |j                  «       }t        «       }t        «       }|j                  D ]‹  }| j                  |«      r?|j                  D ]/  }|j
                  |   }|j                  sŒ|j                  |«       Œ1 ŒS| j                  |«      sŒe|j                  D ]  }||v sŒ|j                  |«       Œ Œ� t        |«      dk(  ryt        j                  d|› �«       y)z;
        Pipeline may need multiple forward before
        r   Naˆ  The pipeline requires multiple forward calculations before backward, so when the persistable var is changed in the forward, it may cause errors in the backward calculation who using this persistable var. However, some backward op don't need this var(NoNeedBufferVars), there will be no error at this time.
So please check these persistable vars which changed in forward and used in backward:
)r®   rp   rQ   rž   rS   r�   ri   r|   r    r{   rx   ÚwarningsÚwarn)r5   r¶   r[   Úpersist_outputÚused_in_backwardr\   rÁ   rT   s           r:   Ú_check_pipeline_persist_varz-PipelineOptimizer._check_pipeline_persist_varü  sØ   € ð ×$Ñ$Ó&ˆä›ˆÜ›5ÐØ—)”)ˆBØ×"Ñ" 2Ô&Ø "× 3Ô 3�HØŸ*™* XÑ.�CØ—“Ø&×*Ñ*¨8Õ4ñ !4ð ×%Ñ% bÕ)Ø "× 2Ô 2�HØ >Ò1Ø(×,Ñ,¨XÕ6ñ !3ð ô ÐÓ  AÒ%ØÜ�‰ð.ð /?Ð-?ðAõ	
r<   c                 ó*	  — |j                   }|| _        |j                  }|€
t        «       }|j                  }|sJ d«       ‚g d¢}|D ]  }	|	|v rŒJ d|	› d�«       ‚ |d   | _        |d   | _        |d   | _        |d   | _        |d	   | _	        |d
   | _
        |d   | _        |d   | _        |j                  dd«      | _        | j                  dk\  sJ ‚d| j                  cxk  r| j                  k  sJ ‚ J ‚| j                  j!                  ||||«      \  }
}| j"                  j$                  | _        | j'                  |«      \  | _        | _        | j-                  |«       | j/                  |«      }d„ }t1        |t3        |«      ¬«      }||k(  sJ d«       ‚| j5                  |«       |j                  }| j7                  ||«      }|D ]"  }| j9                  |j;                  «       |«       Œ$ t=        j>                  dd «      rBtA        t=        j>                  d«      «      | _        | j
                  tC        |«      k  s%J d«       ‚| xj
                  tC        |«      z  c_        | jE                  || j
                     «       | jG                  ||«       g }|D ][  }tA        |jI                  d«      d   «      }tK        jL                  «       sŒ5|jO                  tK        jP                  |dz  «      «       Œ] | jS                  || j
                  «      }d|i|_        || j
                     j;                  «       }| j                  s| jU                  |«       | j                  sB| jW                  |«       |jY                  «        | j[                  |«       |jY                  «        tK        jL                  «       rtA        t=        j>                  dd«      «      }| j]                  || j
                     «       | j_                  || j
                     «       dd| j
                  tC        |«      | j                  tC        |«      || j
                     || j
                     d| j`                  | jb                  dœ|_        |
||| jd                  | jf                  fS )NzPlease use pipeline with fleet.)Ú
local_rankr?  rA  rO   rY   r…   rD  rE  z&Please use pipeline with fleet to use r   r  r?  rA  r…   rO   rY   rD  rE  Úscale_gradientFr   r   c                 ó�   — t        | j                  d«      d   «      }t        |j                  d«      d   «      }||k  ry||kD  ryy)NrÆ   r   rí   r   )r•   rÇ   )Údevice1Údevice2Údev1_idÚdev2_ids       r:   Ú
device_cmpz.PipelineOptimizer.minimize.<locals>.device_cmpL  sK   € Ü˜'Ÿ-™-¨Ó,¨QÑ/Ó0ˆGÜ˜'Ÿ-™-¨Ó,¨QÑ/Ó0ˆGØ˜Ò ØØ˜7Ò"Øàr<   )rº   z`With pipeline parallelism, you must use gpu devices one after another in the order of their ids.ÚPADDLE_MANUAL_PIPELINE_STAGEzTManually specified pipeline stage must be less than total number of pipeline stages.rÆ   rÈ   ÚFLAGS_selected_gpusÚ0ÚPipelineTrainerÚSectionrí   )ÚtrainerÚdevice_workerÚpipeline_stageÚnum_pipeline_stagesr?  Úinner_parallelismÚsection_programÚplaceÚplace_idÚ
sync_stepsr6   r7   )4r[   Úorigin_main_blockr¶   r
   Ú_pipeline_optr  r?  rA  r…   rO   rY   rD  rE  Úgetr  r!   Úminimizer"   r0   rû  r3   r4   r  r  Úsortedr   r[  r»   r‘   r®   ÚosÚgetenvr•   rx   r  rà  rÇ   r   r   ru   Ú	CUDAPlacerÍ   ra  rh  r†   rˆ  r	  r  r$   r%   r1   r2   )r5   ÚlossrÈ   Úparameter_listÚno_grad_setr›  r²   Úpipeline_optÚrequired_keysrº   Úoptimize_opsÚparams_gradsr  r  Úsorted_device_listr¹   ÚpÚ
place_listÚdevÚ	dev_indexrÊ   Ú
real_blockr%  s                          r:   r*  zPipelineOptimizer.minimize  sw  € ð —Z‘Zˆ
Ø!+ˆÔØ!×)Ñ)ˆØÐ"Ü5Ó7ˆOà#×1Ñ1ˆÙÐ>Ð>Ó>ˆ|ò	
ˆó !ˆCà�|Ò#ð?à7¸°u¸AÐ>ó?Ø#ð !ð ' |Ñ4ˆŒØ)¨/Ñ:ˆÔØ ,Ð-?Ñ @ˆÔØ(¨Ñ8ˆÔØ# IÑ.ˆŒØ*Ð+;Ñ<ˆÔØ% kÑ2ˆŒØ# IÑ.ˆŒØ*×.Ñ.Ð/?ÀÓGˆÔØ�~‰~ Ò"Ð"Ð"Ø�D—L‘LÔ1 4§>¡>Ò1Ð1Ñ1Ð1Ð1à%)§_¡_×%=Ñ%=Ø�/ >°;ó&
Ñ"ˆ�lð "&×!7Ñ!7×!IÑ!IˆÔð
 ×'Ñ'¨
Ó3ñ	
ØÔ!ØÔ ð 	× Ñ  Ô,Ø×,Ñ,¨ZÓ8ˆò	ô $ K´ZÀ
Ó5KÔLÐØ! [Ò0ð 	
ð1ó	
Ð0ð
 	×0Ñ0°Ô<ð "×)Ñ)ˆØ×*Ñ*¨<¸ÓEˆÛˆAØ×Ñ˜aŸn™nÓ.°
Õ;ð ô �9‰9Ð3°TÔ:Ü!¤"§)¡)Ð,JÓ"KÓLˆDŒOØ—?‘?¤S¨Ó%5Ò5ð ðóÐ5ð �OŠOœs ;Ó/Ñ/�Oà×(Ñ(¨°d·o±oÑ)FÔGð 	×Ñ˜Z¨Ô6àˆ
ÛˆCÜ˜CŸI™I c›N¨1Ñ-Ó.ˆIÜ×)Ñ)Õ+Ø×!Ñ!¤$§.¡.°¸Q±Ó"?Õ@ð ð #×9Ñ9Ø˜TŸ_™_ó
Ðð
 Ð2ð)
ˆÔ%ð " $§/¡/Ñ2×?Ñ?ÓAˆ
Ø×"Ò"Ø×#Ñ# JÔ/Ø× Ò ð ×*Ñ*¨:Ô6Ø×%Ñ%Ô'Ø×&Ñ& zÔ2Ø×%Ñ%Ô'ä×%Ñ%Ô'Üœ2Ÿ9™9Ð%:¸CÓ@ÓAˆHð 	×Ñ˜<¨¯©Ñ8Ô9ð 	×(Ñ(¨°d·o±oÑ)FÔGð )Ø&Ø"Ÿo™oÜ#& {Ó#3Ø!×/Ñ/Ü!$ [Ó!1Ø+¨D¯O©OÑ<Ø §¡Ñ0Ø ØØ $× 6Ñ 6Ø!%×!8Ñ!8ñ&
ˆÔ"ð ØØØ×ÑØ×Ñð
ð 	
r<   )r   r   r“   )FNN)NNN).Ú__name__Ú
__module__Ú__qualname__Ú__doc__r;   rb   r‘   rš   rž   r    r£   rr   rª   r»   rÃ   rÍ   rÖ   rÛ   rá   rç   r„   rñ   ró   r÷   r  r  r  r[  ra  rh  rˆ  r¸  rq  r—  r™  rà  râ  rò  rz   r  rù  rû  r  r	  r  r*  rÉ  r<   r:   r   r   "   s  „ ñ9óv)$òZ6òpUòn
ò
ò

ò
Jò
ò

ò$òLò#ò8ò(ò.óò&5ò
1ò-òò xEòt<ò.6òpgòR	ò$:ð& KOó]%ò~]8ð@ 37ó<*ò|ò$
ò(9ò"ò|ò|'ò
(ò
=ò1ò"*òX7òr
ð> LPôW
r<   r   )r,  r  Úcollectionsr   Ú	functoolsr   r   ÚnumpyrB  r   Úpaddle.baser   r   Úpaddle.base.frameworkr   r	   r
   r   Ú__all__r   rÉ  r<   r:   Ú<module>rF     s9   ðó 
Û Ý #ß (ã ã ß )÷ó ð €÷O
ò O
r<   