Ë
    •\;jB`  ã                   ón   — d dl Z d dlZd dlmZ d dlmZ d dlmZ ddlmZm	Z	 ddl
mZ g Z G d	„ d
e«      Zy)é    N)Ú_C_ops)Úin_dynamic_or_pir_mode)ÚL2Decayé   )ÚcoreÚ	frameworké   )Ú	Optimizerc                   ól   ‡ — e Zd ZdZdZ	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zd„ Zd„ Zdˆ fd„	Zd„ Z	d„ Z
d	„ Zd
„ Zˆ xZS )ÚMomentuma…  

    Simple Momentum optimizer with velocity state

    This optimizer has a flag for Nestrov Momentum.

    The update equations are as follows:

    .. math::

        & velocity = mu * velocity + gradient

        & if (use\_nesterov):

        &\quad   param = param - (gradient + mu * velocity) * learning\_rate

        & else:

        &\quad   param = param - learning\_rate * velocity

    Parameters:

        learning_rate (float|Tensor|LearningRateDecay, optional): The learning rate used to update ``Parameter``.
            It can be a float value, a ``Tensor`` with a float type or a LearningRateDecay. The default value is 0.001.
        momentum (float): Momentum factor. The default value is 0.9.
        parameters (list|tuple, optional): List|Tuple of ``Tensor`` to update to minimize ``loss``. \
            This parameter is required in dygraph mode. And you can specify different options for \
            different parameter groups such as the learning rate, weight decay, etc, \
            then the parameters are list of dict. Note that the learning_rate in parameter groups \
            represents the scale of base learning_rate. \
            The default value is None in static graph mode, at this time all parameters will be updated.
        weight_decay (float|WeightDecayRegularizer, optional): The strategy of regularization. \
            It can be a float value as coeff of L2 regularization or \
            :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
            If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
            the regularization setting here in optimizer will be ignored for this parameter. \
            Otherwise, the regularization setting here in optimizer will take effect. \
            Default None, meaning there is no regularization.
        grad_clip (GradientClipBase, optional): Gradient clipping strategy, it's an instance of
            some derived class of ``GradientClipBase`` . There are three clipping strategies
            ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` ,
            :ref:`api_paddle_nn_ClipGradByValue` ). Default None, meaning there is no gradient clipping.
        multi_precision (bool, optional): Whether to use multi-precision during weight updating. Default is false.
        rescale_grad (float, optional): Multiply the gradient with `rescale_grad` before updating. \
            Often choose to be ``1.0/batch_size``.
        use_multi_tensor (bool, optional): Whether to use multi-tensor strategy to update all parameters at once . Default is false.
        name (str, optional): The default value is None. Normally there is no need for user
                to set this property. For more information, please refer to
                :ref:`api_guide_Name` .

    Examples:
        .. code-block:: python

            >>> import paddle

            >>> inp = paddle.uniform([10, 10], dtype="float32", min=-0.1, max=0.1)
            >>> linear = paddle.nn.Linear(10, 10)
            >>> inp = paddle.to_tensor(inp)
            >>> out = linear(inp)
            >>> loss = paddle.mean(out)
            >>> beta1 = paddle.to_tensor([0.9], dtype="float32")
            >>> beta2 = paddle.to_tensor([0.99], dtype="float32")
            >>> momentum = paddle.optimizer.Momentum(learning_rate=0.1, parameters=linear.parameters(), weight_decay=0.01)
            >>> back = out.backward()
            >>> momentum.step()
            >>> momentum.clear_grad()

            >>> # Note that the learning_rate of linear_2 is 0.01.
            >>> linear_1 = paddle.nn.Linear(10, 10)
            >>> linear_2 = paddle.nn.Linear(10, 10)
            >>> inp = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1)
            >>> out = linear_1(inp)
            >>> out = linear_2(out)
            >>> loss = paddle.mean(out)
            >>> momentum = paddle.optimizer.Momentum(
            ...     learning_rate=0.1,
            ...     parameters=[{
            ...         'params': linear_1.parameters()
            ...     }, {
            ...         'params': linear_2.parameters(),
            ...         'weight_decay': 0.001,
            ...         'learning_rate': 0.1
            ...     }],
            ...     weight_decay=0.01,
            ...     momentum=0.9
            ... )
            >>> out.backward()
            >>> momentum.step()
            >>> momentum.clear_grad()

    Úvelocityc                 óD  •— |€t        d«      ‚|€t        d«      ‚d„ }t        |t        «      rTt        |d   t        «      rA|D ]<  }d|v r|d   n|}| j	                  |«      \  }}||d<   ||d<    ||«      rd n|}||d<   Œ>  ||«      rd n|}t
        ‰| �  |||||
¬«       d	| _        || _        t        |«      | _
        | j	                  |«      \  | _        | _        || _        || _        i | _        |||| j                  | j                  d
œ| _        |	| _        | j"                  ry| j%                  «       | _        | j%                  «       | _        | j%                  «       | _        d | j*                  d<   | j%                  «       | _        | j%                  «       | _        y y )Nzlearning_rate is not setzmomentum is not setc                 ó.   — t        | t        t        f«      S ©N)Ú
isinstancer   Úfloat)Úregulars    úbG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/optimizer/momentum.pyÚ<lambda>z#Momentum.__init__.<locals>.<lambda>Œ   s   € ¤J¨w¼Ä%Ð8HÔ$Ió    r   Úweight_decayÚregularization_methodÚregularization_coeff)Úlearning_rateÚ
parametersr   Ú	grad_clipÚnameÚmomentum)r   Úuse_nesterovÚrescale_gradr   r   ÚFP32_LODTensor)Ú
ValueErrorr   ÚlistÚdictÚ_update_regularizationÚsuperÚ__init__ÚtypeÚ	_momentumÚboolÚ_use_nesterovÚ_regularization_methodÚ_regularization_coeffÚ_multi_precisionÚ_rescale_gradÚ_master_weightsÚ_default_dictÚ_use_multi_tensorÚ_create_multi_tensor_dictÚ_param_dictÚ_velocity_dictÚ_master_weight_dictÚ_regularization_method_dictÚ_regularization_coeff_dict)Úselfr   r   r   r   r   r   Úmulti_precisionr    Úuse_multi_tensorr   Ú	predicateÚparam_groupÚdecayÚ
reg_methodÚ	reg_coeffÚ
py_regularÚ	__class__s                    €r   r'   zMomentum.__init__z   sÝ  ø€ ð Ð ÜÐ7Ó8Ð8ØÐÜÐ2Ó3Ð3áIˆ	Ü�j¤$Ô'Ü˜* Q™-¬Ô.Û#-�Kð *¨[Ñ8ð $ NÒ3à)ð ð
 -1×,GÑ,GÈÓ,NÑ)�J 	Ø;E�KÐ 7Ñ8Ø:C�KÐ 6Ñ7Ù)2°5Ô)9¡¸u�JØ2<�K Ò/ð $.ñ ' |Ô4‘T¸,ˆ
Ü‰ÑØ'Ø!Ø#ØØð 	ô 	
ð ˆŒ	Ø!ˆŒÜ! ,Ó/ˆÔð ×'Ñ'¨Ó5ñ	
ØÔ'ØÔ&à /ˆÔØ)ˆÔØ!ˆÔð !Ø(Ø(Ø%)×%@Ñ%@Ø$(×$>Ñ$>ñ
ˆÔð "2ˆÔØ×!Ò!Ø#×=Ñ=Ó?ˆDÔØ"&×"@Ñ"@Ó"BˆDÔØ'+×'EÑ'EÓ'GˆDÔ$Ø9=ˆD×$Ñ$Ð%5Ñ6Ø/3×/MÑ/MÓ/OˆDÔ,Ø.2×.LÑ.LÓ.NˆDÕ+ð "r   c                 óv   — d}d}t        |t        «      rd}|j                  }t        |t        «      rd}|}||fS )NÚ ç        Úl2_decay)r   r   Ú_coeffr   )r9   r   r?   r@   s       r   r%   zMomentum._update_regularization¾   sE   € Øˆ
Øˆ	ä�l¤GÔ,Ø#ˆJØ$×+Ñ+ˆIÜ�l¤EÔ*Ø#ˆJØ$ˆIØ˜9Ð$Ð$r   c                 óä  — t        |t        j                  t        j                  j                  f«      sJ ‚t        |t
        «      r| j                  |«      }|D �]  }|j                  | j                  v rŒ| j                  rn| j                  |j                  «      rS| j                  |«      }| j                  | j                  |«       | j                  j                  |j                  «       Œ—| j                  |j                  «      r!| j                  st!        j"                  d«       | j                  | j                  |«       | j                  j                  |j                  «       �Œ y)zD
        if framework.in_dynamic_mode():
            return
        zœAccumulating with FP16/BF16 in optimizer can lead to poor accuracy or slow convergence.Consider using multi_precision=True option of the Momentum optimizer.N)r   r   ÚBlockÚpaddleÚpirr$   Ú_update_param_groupr   Ú_already_create_accumulaterr.   Ú_is_dtype_fp16_or_bf16ÚdtypeÚ_create_master_weightÚ_add_accumulatorÚ_velocity_acc_strÚaddÚwarningsÚwarn)r9   Úblockr   ÚpÚmaster_ps        r   Ú_create_accumulatorszMomentum._create_accumulatorsÊ   s  € ô
 ˜%¤)§/¡/´6·:±:×3CÑ3CÐ!DÔEÐEÐEä�j¤$Ô'Ø×1Ñ1°*Ó=ˆJäˆAØ�v‰v˜×9Ñ9Ñ9ØØ×$Ò$¨×)DÑ)DÀQÇWÁWÔ)MØ×5Ñ5°aÓ8�Ø×%Ñ% d×&<Ñ&<¸hÔGØ×0Ñ0×4Ñ4°Q·V±VÔ<Øà×+Ñ+¨A¯G©GÔ4Ø×-Ò-ä—‘ð\ôð ×!Ñ! $×"8Ñ"8¸!Ô<Ø×,Ñ,×0Ñ0°·±Ö8ñ# r   c                 óv   •— t        |d«      rt        |j                  t        «      r|S t        ‰| �  |||«      S )zpCreate and add backward regularization Operators

        Function helper of append_regularization_ops.
        Úregularizer)Úhasattrr   r[   r   r&   Ú_create_regularization_of_grad)r9   ÚparamÚgradÚregularizationrB   s       €r   r]   z'Momentum._create_regularization_of_gradç   sA   ø€ ô �5˜-Ô(¬ZØ×Ñœwô.
ð ˆKÜ‰wÑ5Ø�4˜ó
ð 	
r   c                 ó  — t        |t        j                  «      sJ ‚t        |t        «      r| j	                  |«      }| j                  | j                  |d   «      }| j                  |«      }|d   }| j                  }| j                  }t        |d«      rCt        |j                  t        «      rd}|j                  j                  }n|j                  �d}d}| j                  xr | j                  |d   j                   «      }|r| j"                  |d   j$                     nd }	t'        «       rgt        |t        «      r| j)                  |d   «       t+        j,                  |d   |d   |||	| j.                  | j0                  |||| j2                  «      S | j.                  | j0                  |||| j2                  dœ}
|d   g|d   g|g|gd	œ}|d   g|gd
œ}|r
|	|d<   |	|d<   |j5                  | j6                  |||
d¬«      }|S )Nr   r[   rF   rD   rE   r   r	   )Úmur   r   r   r:   r    ©ÚParamÚGradÚVelocityÚLearningRate©ÚParamOutÚVelocityOutÚMasterParamÚMasterParamOutT©r(   ÚinputsÚoutputsÚattrsÚstop_gradient)r   r   rI   r$   rL   Ú_get_accumulator_masterrR   Ú_create_param_lrr,   r-   r\   r[   r   rG   r.   rN   rO   r0   r   r   r%   r   Ú	momentum_r)   r+   r/   Ú	append_opr(   )r9   rV   Úparam_and_gradÚvelocity_accÚlrr^   r   r   Úfind_masterÚmaster_weightrp   rn   ro   Úmomentum_ops                 r   Ú_append_optimize_opzMomentum._append_optimize_opö   sM  € Ü˜%¤§¡Ô1Ð1Ð1Ü�n¤dÔ+Ø!×5Ñ5°nÓEˆNà×3Ñ3Ø×"Ñ" N°1Ñ$5ó
ˆð ×"Ñ" >Ó2ˆð ˜qÑ!ˆØ $× ;Ñ ;ÐØ#×9Ñ9ÐÜ�5˜-Ô(ä˜%×+Ñ+¬WÔ5Ø(2Ð%Ø',×'8Ñ'8×'?Ñ'?Ñ$à×"Ñ"Ð.Ø(*Ð%Ø'*Ð$à×+Ñ+ò 
°×0KÑ0KØ˜1Ñ×#Ñ#ó1
ˆñ
 ð × Ñ  °Ñ!2×!7Ñ!7Ò8àð 	ô "Ô#Ü˜.¬$Ô/Ø×+Ñ+¨N¸>Ñ,JÔKÜ×#Ñ#Ø˜qÑ!Ø˜qÑ!ØØØØ—‘Ø×"Ñ"Ø%Ø$ØØ×"Ñ"óð ð —n‘nØ $× 2Ñ 2Ø)>Ø(<Ø#.Ø $× 2Ñ 2ñˆEð )¨Ñ+Ð,Ø'¨Ñ*Ð+Ø)˜NØ!# ñ	ˆFð ,¨AÑ.Ð/Ø ,˜~ñˆGñ
 Ø(5��}Ñ%Ø,9�Ð(Ñ)ð  Ÿ/™/Ø—Y‘YØØØØ"ð *ó ˆKð Ðr   c                 óˆ  — | j                  ||«       |D �]*  }| j                  | j                  |«      }| j                  }| j                  }t        |d«      rCt        |j                  t        «      rd}|j                  j                  }n|j                  �d}d}|j                  t        j                  k(  r†| j                  d   |   j                  |«       | j                  d   |   j                  |«       | j                   d   |   j                  |«       | j"                  d   |   j                  |«       �Œ*| j%                  |j                  «      rÝ| j                  d   |   j                  |«       | j                  d   |   j                  |«       | j&                  r9| j(                  d   |   j                  | j*                  |j,                     «       nd| j(                  d   |<   | j                   d   |   j                  |«       | j"                  d   |   j                  |«       �Œ"t/        d«      ‚ y)	a¯  
        All parameters used for optimizer (such as: parameters, master_weight, velocity_acc for momentum) calculations are grouped into a python list by data type (float16, bf16, float32).
        This function will be overridden in the corresponding optimizer file.

        Args:
            target_block: the block in which the loss tensor is present
            parameters: list of parameter tensors for the optimizer
        r[   rF   NrD   rE   r!   ÚFP16_LODTensorz\Now multi_tensor_momentum only support fp32, fp16 or bf16 parameters and grad is LOD_TENSOR.)rY   rr   rR   r,   r-   r\   r   r[   r   rG   rO   rJ   Úfloat32r4   Úappendr5   r7   r8   rN   r.   r6   r0   r   r"   )r9   Útarget_blockr   Úparam_group_idxr^   rw   r   r   s           r   Ú_multi_tensor_initzMomentum._multi_tensor_initL  sA  € ð 	×!Ñ! ,°
Ô;ÜˆEØ×7Ñ7Ø×&Ñ&¨óˆLð %)×$?Ñ$?Ð!Ø#'×#=Ñ#=Ð Ü�u˜mÔ,ä˜e×/Ñ/´Ô9Ø,6Ð)Ø+0×+<Ñ+<×+CÑ+CÑ(Ø×&Ñ&Ð2Ø,.Ð)Ø+.Ð(Ø�{‰{œfŸn™nÒ,Ø× Ñ Ð!1Ñ2°?ÑC×JÑJØôð ×#Ñ#Ð$4Ñ5°oÑF×MÑMØ ôð ×0Ñ0Ð1AÑBØ#ñç‘&Ð.Ô/Ø×/Ñ/Ð0@ÑAØ#ñç‘&Ð-Ö.Ø×,Ñ,¨U¯[©[Ô9Ø× Ñ Ð!1Ñ2°?ÑC×JÑJØôð ×#Ñ#Ð$4Ñ5°oÑF×MÑMØ ôð ×(Ò(Ø×,Ñ,Ð-=Ñ>Ø'ñç‘f˜T×1Ñ1°%·*±*Ñ=Õ>ð ð ×,Ñ,Ð-=Ñ>Ø'ñð ×0Ñ0Ð1AÑBØ#ñç‘&Ð.Ô/Ø×/Ñ/Ð0@ÑAØ#ñç‘&Ð-Ö.ä Øróð ñe  r   c                 ó  — t        |t        j                  «      sJ ‚g g dœ}g g dœ}t        |t        «      �rE|D �]=  }|d   €Œ
|d   j                  du sŒ|d   j
                  t        j                  k(  rq|d   j                  t        j                  j                  j                  k(  r=|d   j                  |d   «       | j                  |«      }|d   j                  |«       Œ­| j                  |d   j
                  «      sŒÌ|d   j                  t        j                  j                  j                  k(  s�Œ|d   j                  |d   «       | j                  |«      }|d   j                  |«       �Œ@ �n—|d   D �]Ž  }|d   €Œ
|d   j                  du sŒi }||d<   |j!                  |j#                  «       D �	�
ci c]  \  }	}
|	dk7  r|	|
“Œ c}
}	«       | j%                  |«      }|d   j
                  t        j                  k(  rq|d   j                  t        j                  j                  j                  k(  r=|d   j                  |d   «       | j                  |«      }|d   j                  |«       Œý| j                  |d   j
                  «      s�Œ|d   j                  t        j                  j                  j                  k(  s�ŒS|d   j                  |d   «       | j                  |«      }|d   j                  |«       �Œ‘ ddg}|D �]j  }t'        | j(                  |   |   «      dkD  sŒ#| j*                  xr |dk(  }| j,                  |   }|�||   nd}t/        «       �r>| j1                  d	«      }|rRt        |t        j2                  j4                  t        j6                  j8                  f«      sŒ«| j;                  d	d
«       Œ¾t        |t        j2                  j4                  t        j6                  j8                  f«      r| j;                  d	d«       t=        j>                  | j(                  |   |   ||   | j@                  |   |   ||   || jB                  | jD                  | jF                  |   |   | jH                  |   |   || jJ                  «      \  }}}�Œ—| j(                  |   |   ||   | j@                  |   |   ||   dœ}| j(                  |   |   | j@                  |   |   dœ}| jB                  | jD                  | jF                  |   |   | jH                  |   |   dœ}|r/| j,                  |   |   |d<   | j,                  |   |   |d<   ||d<   |jM                  d|||d
¬«       �Œm yc c}
}	w )zM
        For Multi Tensor, append optimize merged_operator to block.
        )r!   r~   r	   Nr   Fr!   r~   ÚparamsÚ	found_infTrc   rh   )rb   r   r   r   rk   rl   r:   Úmerged_momentumrm   )'r   r   rI   r#   rq   rO   rJ   r   r(   r   ÚVarDescÚVarTypeÚ
LOD_TENSORr€   rs   rN   ÚupdateÚitemsrL   Úlenr4   r.   r6   r   Ú_get_auxiliary_varÚeagerÚTensorrK   ÚOpResultÚ_set_auxiliary_varr   Úmerged_momentum_r5   r)   r+   r7   r8   r/   ru   )r9   r�   Úparameters_and_gradsr‚   Ú	grad_dictÚlr_dictrv   rx   Úparam_grad_dictÚkÚvÚmulti_tensor_listÚkeyry   rz   r†   Ú_rn   ro   rp   s                       r   Ú _append_optimize_multi_tensor_opz)Momentum._append_optimize_multi_tensor_opŒ  s¾  € ô ˜,¬	¯©Ô8Ð8Ð8à')¸RÑ@ˆ	Ø%'¸2Ñ>ˆäÐ*¬DÕ1Ü"6�Ø! !Ñ$Ð,ØØ! !Ñ$×2Ñ2°eÒ;à& qÑ)×/Ñ/´6·>±>ÒAØ*¨1Ñ-×2Ñ2ÜŸ<™<×/Ñ/×:Ñ:ò;ð "Ð"2Ñ3×:Ñ:¸>È!Ñ;LÔMØ!×2Ñ2°>ÓB˜ØÐ 0Ñ1×8Ñ8¸Õ<à×3Ñ3°NÀ1Ñ4E×4KÑ4KÕLØ*¨1Ñ-×2Ñ2ÜŸ<™<×/Ñ/×:Ñ:ô;ð "Ð"2Ñ3×:Ñ:¸>È!Ñ;LÔMØ!×2Ñ2°>ÓB˜ØÐ 0Ñ1×8Ñ8¸Ö<ò' #7ð* #7°xÕ"@�Ø! !Ñ$Ð,ØØ! !Ñ$×2Ñ2°eÒ;Ø&(�OØ0>�O HÑ-Ø#×*Ñ*ð )=×(BÑ(BÔ(Dôá(D¡  1Ø  Hš}ð ˜q™DØ(Dòôð &*×%=Ñ%=¸oÓ%N�Nà& qÑ)×/Ñ/´6·>±>ÒAØ*¨1Ñ-×2Ñ2ÜŸ<™<×/Ñ/×:Ñ:ò;ð "Ð"2Ñ3×:Ñ:¸>È!Ñ;LÔMØ!×2Ñ2°>ÓB˜ØÐ 0Ñ1×8Ñ8¸Õ<à×3Ñ3°NÀ1Ñ4E×4KÑ4KÖLØ*¨1Ñ-×2Ñ2ÜŸ<™<×/Ñ/×:Ñ:ô;ð "Ð"2Ñ3×:Ñ:¸>È!Ñ;LÔMØ!×2Ñ2°>ÓB˜ØÐ 0Ñ1×8Ñ8¸Ö<ð; #Að> .Ð/?Ð@ÐÜ$ˆCÜ�4×#Ñ# CÑ(¨Ñ9Ó:¸QÓ>Ø"×3Ñ3ÒO¸Ð?OÑ8O�à $× 8Ñ 8¸Ñ =�ð %Ð0ð " /Ò2àð ô *Õ+Ø $× 7Ñ 7¸Ó D�IÙ Ü%Ø%¬¯
©
×(9Ñ(9¼6¿:¹:×;NÑ;NÐ'Oõð !×3Ñ3°KÀÕFä%Ø%¬¯
©
×(9Ñ(9¼6¿:¹:×;NÑ;NÐ'Oôð !×3Ñ3°KÀÔGÜ"(×"9Ñ"9Ø ×,Ñ,¨SÑ1°/ÑBØ% c™NØ ×/Ñ/°Ñ4°_ÑEØ# C™LØ)Ø ŸN™NØ ×.Ñ.Ø ×<Ñ<¸SÑAØ /ñð !×;Ñ;¸CÑ@Ø /ñð (Ø ×.Ñ.ó#™˜˜1šað& "&×!1Ñ!1°#Ñ!6°Ñ!GØ )¨#¡Ø$(×$7Ñ$7¸Ñ$<¸_Ñ$MØ(/°©ñ	�Fð %)×$4Ñ$4°SÑ$9¸/Ñ$JØ'+×':Ñ':¸3Ñ'?Ø+ñ(ñ�Gð #Ÿn™nØ(,×(:Ñ(:Ø15×1QÑ1QØñ2ð ,ñ2ð
 15×0OÑ0OØñ1à)ñ1+ñ�Eñ #Ø04×0HÑ0HÈÑ0MØ+ñ1˜˜}Ñ-ð 59×4LÑ4LØñ5à)ñ5+˜Ð 0Ñ1ð 4?˜Ð/Ñ0Ø ×*Ñ*Ø.Ø%Ø 'Ø#Ø&*ð +ö ñS %ùó3s   ÇV	c                 ó�  — |j                  d| j                  d   «      | _        |j                  d| j                  d   «      | _        |j                  d| j                  d   «      | _        |j                  d| j                  d   «      | _        |j                  d| j                  d   «      | _        |j                  d«      }|S )Nr   r   r    r   r   r…   )Úgetr1   r)   r+   r/   r,   r-   )r9   r   s     r   rL   zMomentum._update_param_group!  sÆ   € Ø#Ÿ™Ø˜×*Ñ*¨:Ñ6ó
ˆŒð (Ÿ^™^Ø˜D×.Ñ.¨~Ñ>ó
ˆÔð (Ÿ^™^Ø˜D×.Ñ.¨~Ñ>ó
ˆÔð '1§n¡nØ# T×%7Ñ%7Ð8OÑ%Pó'
ˆÔ#ð &0§^¡^Ø" D×$6Ñ$6Ð7MÑ$Nó&
ˆÔ"ð  —^‘^ HÓ-ˆ
ØÐr   )
gü©ñÒMbP?gÍÌÌÌÌÌì?NFNNFg      ð?FNr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rR   r'   r%   rY   r]   r|   rƒ   r�   rL   Ú__classcell__)rB   s   @r   r   r      sb   ø„ ñZðv #Ðð ØØØØØØØØØõBOòH
%ò9õ:
òTòl>ò@Söjr   r   )rT   rJ   r   Úpaddle.frameworkr   Úpaddle.regularizerr   Úbaser   r   Ú	optimizerr
   Ú__all__r   © r   r   Ú<module>r«      s/   ðó ã Ý Ý 3Ý &ç "Ý  à
€ôVˆyõ Vr   