Ë
    •\;j¡(  ã                   ób   — d dl Z d dlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 g Z
 G d	„ d
e	«      Zy)é    N)Ú_C_opsé   )Ú	framework)Úno_grad)Úin_dynamic_modeé   )Ú	Optimizerc                   óL   ‡ — e Zd ZdZdZdZ	 	 	 	 	 	 	 dˆ fd„	Zd„ Zd„ Zd„ Z	ˆ xZ
S )	ÚAdadeltaaU  
    **Notes: This API does not support sparse parameter optimization.**

    Adadelta Optimizer. Please refer to this for details:
    `ADADELTA: AN ADAPTIVE LEARNING RATE METHOD <https://arxiv.org/abs/1212.5701>`_.

    The update is done as follows:

    .. math::

        E(g_t^2) &= \rho * E(g_{t-1}^2) + (1-\rho) * g^2

        learning\_rate &= \sqrt{ ( E(dx_{t-1}^2) + \epsilon ) / ( E(g_t^2) + \epsilon ) }

        E(dx_t^2) &= \rho * E(dx_{t-1}^2) + (1-\rho) * (-g*learning\_rate)^2

    Args:
        learning_rate (float|Tensor|LearningRateDecay, optional): The learning rate used to update ``Parameter``.
            It can be a float value, a ``Tensor`` with a float type or a LearningRateDecay. The default value is 0.001.
        epsilon (float): a small float number for numeric stability. Default 1.0e-6.
        rho (float): a floating point value indicating the decay rate. Default 0.95.
        parameters (list|tuple, optional): List/Tuple of ``Tensor`` to update to minimize ``loss``. \
            This parameter is required in dygraph mode. And you can specify different options for \
            different parameter groups such as the learning rate, weight decay, etc, \
            then the parameters are list of dict. Note that the learning_rate in paramter groups \
            represents the scale of base learning_rate. \
            The default value is None in static graph mode, at this time all parameters will be updated.
        weight_decay (float|WeightDecayRegularizer, optional): The strategy of regularization. \
            It canbe a float value as coeff of L2 regularization or \
            :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
            If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
            the regularization setting here in optimizer will be ignored for this parameter. \
            Otherwise, the regularization setting here in optimizer will take effect. \
            Default None, meaning there is no regularization.
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of
            some derived class of ``GradientClipBase`` . There are three cliping strategies
            ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` ,
            :ref:`api_paddle_nn_ClipGradByValue` ). Default None, meaning there is no gradient clipping.
        name (str, optional): The default value is None. Normally there is no need for user
                to set this property. For more information, please refer to
                :ref:`api_guide_Name` .

    Examples:
        .. code-block:: python

            >>> import paddle

            >>> inp = paddle.uniform([10, 10], dtype="float32", min=-0.1, max=0.1)
            >>> linear = paddle.nn.Linear(10, 10)
            >>> out = linear(inp)
            >>> loss = paddle.mean(out)
            >>> beta1 = paddle.to_tensor([0.9], dtype="float32")
            >>> beta2 = paddle.to_tensor([0.99], dtype="float32")
            >>> adadelta = paddle.optimizer.Adadelta(learning_rate=0.1, parameters=linear.parameters(), weight_decay=0.01)
            >>> back = out.backward()
            >>> adadelta.step()
            >>> adadelta.clear_grad()

            >>> # Note that the learning_rate of linear_2 is 0.01.
            >>> linear_1 = paddle.nn.Linear(10, 10)
            >>> linear_2 = paddle.nn.Linear(10, 10)
            >>> inp = paddle.uniform(shape=[10, 10], min=-0.1, max=0.1)
            >>> out = linear_1(inp)
            >>> out = linear_2(out)
            >>> loss = paddle.mean(out)
            >>> adadelta = paddle.optimizer.Adadelta(
            ...     learning_rate=0.1,
            ...     parameters=[{
            ...         'params': linear_1.parameters()
            ...     }, {
            ...         'params': linear_2.parameters(),
            ...         'weight_decay': 0.001,
            ...         'learning_rate': 0.1,
            ...     }],
            ...     weight_decay=0.01)
            >>> out.backward()
            >>> adadelta.step()
            >>> adadelta.clear_grad()

    Ú_avg_squared_gradÚ_avg_squared_updatec                 óÖ   •— |€t        d«      ‚|€t        d«      ‚|€t        d«      ‚t        ‰| �	  |||||¬«       d| _        i | _        d| _        || _        || _        ||dœ| _        y )Nzlearning_rate is not set.zepsilon is not set.zrho is not set.)Úlearning_rateÚ
parametersÚweight_decayÚ	grad_clipÚnameFÚadadelta)ÚepsilonÚrho)	Ú
ValueErrorÚsuperÚ__init__Ú_multi_precisionÚ_master_weightsÚtypeÚ_epsilonÚ_rhoÚ_default_dict)	Úselfr   r   r   r   r   r   r   Ú	__class__s	           €úbG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/optimizer/adadelta.pyr   zAdadelta.__init__p   s‘   ø€ ð Ð ÜÐ8Ó9Ð9Øˆ?ÜÐ2Ó3Ð3Øˆ;ÜÐ.Ó/Ð/Ü‰ÑØ'Ø!Ø%ØØð 	ô 	
ð !&ˆÔØ!ˆÔØˆŒ	ØˆŒØˆŒ	àØñ
ˆÕó    c                 ó2  — t        |t        j                  «      st        d«      ‚t        |t        «      r|j                  d«      }|D �]K  }|j                  | j                  v rŒ| j                  rŠ| j                  |j                  «      ro| j                  |«      }| j                  | j                  |«       | j                  | j                  |«       | j                  j                  |j                  «       Œ³| j                  |j                  «      r!| j                  st!        j"                  d«       | j                  | j                  |«       | j                  | j                  |«       | j                  j                  |j                  «       �ŒN y )Nú)block is not instance of framework.Block.Úparamsz˜Accumulating with FP16/BF16 in optimizer can lead to poor accuracy or slow convergence.Consider using multi_precision=True option of the Lars optimizer.)Ú
isinstancer   ÚBlockÚ	TypeErrorÚdictÚgetr   Ú_already_create_accumulaterr   Ú_is_dtype_fp16_or_bf16ÚdtypeÚ_create_master_weightÚ_add_accumulatorÚ_avg_squared_grad_acc_strÚ_avg_squared_update_acc_strÚaddÚwarningsÚwarn)r    Úblockr   ÚpÚmaster_ps        r"   Ú_create_accumulatorszAdadelta._create_accumulators‘   s<  € Ü˜%¤§¡Ô1ÜÐGÓHÐHÜ�j¤$Ô'Ø#Ÿ™¨Ó1ˆJäˆAØ�v‰v˜×9Ñ9Ñ9ØØ×$Ò$¨×)DÑ)DÀQÇWÁWÔ)MØ×5Ñ5°aÓ8�Ø×%Ñ% d×&DÑ&DÀhÔOØ×%Ñ%Ø×4Ñ4°hôð ×0Ñ0×4Ñ4°Q·V±VÔ<Øà×+Ñ+¨A¯G©GÔ4Ø×-Ò-ä—‘ðXôð ×!Ñ! $×"@Ñ"@À!ÔDØ×!Ñ! $×"BÑ"BÀAÔFØ×,Ñ,×0Ñ0°·±Ö8ñ+ r#   c                 óZ  — t        |t        «      r| j                  |«      }| j                  | j                  |d   «      }| j                  | j
                  |d   «      }| j                  xr | j                  |d   j                  «      }|r| j                  |d   j                     nd }t        «       rZt        «       5  t        j                  |d   |d   ||| j                  |«      || j                   | j"                  |«	       d d d «       y t        |t$        j&                  «      st)        d«      ‚|d   |d   ||| j                  |«      dœ}|d   ||dœ}|r
||d<   ||d<   |j+                  | j,                  ||| j"                  | j                   |dœd	¬
«      }	|	S # 1 sw Y   y xY w)Nr   r   r%   )ÚParamÚGradÚAvgSquaredGradÚAvgSquaredUpdateÚLearningRate)ÚParamOutÚAvgSquaredGradOutÚAvgSquaredUpdateOutÚMasterParamÚMasterParamOut)r   r   Úmulti_precisionT)r   ÚinputsÚoutputsÚattrsÚstop_gradient)r'   r*   Ú_update_param_groupÚ_get_accumulator_masterr1   r2   r   r-   r.   r   r   r   r   r   Ú	adadelta_Ú_create_param_lrr   r   r   r(   r)   Ú	append_opr   )
r    r6   Úparam_and_gradÚavg_squared_grad_accÚavg_squared_update_accÚfind_masterÚmaster_weightrF   rG   Úadadelta_ops
             r"   Ú_append_optimize_opzAdadelta._append_optimize_op®   sÙ  € Ü�n¤dÔ+Ø!×5Ñ5°nÓEˆNà#×;Ñ;Ø×*Ñ*¨N¸1Ñ,=ó 
Ðð "&×!=Ñ!=Ø×,Ñ,¨n¸QÑ.?ó"
Ðð ×+Ñ+ò 
°×0KÑ0KØ˜1Ñ×#Ñ#ó1
ˆñ
 ð × Ñ  °Ñ!2×!7Ñ!7Ò8àð 	ô ÔÜ•Ü× Ñ Ø" 1Ñ%Ø" 1Ñ%Ø(Ø*Ø×)Ñ)¨.Ó9Ø!Ø—I‘IØ—M‘MØô
÷ ð ä˜e¤Y§_¡_Ô5ÜÐ KÓLÐLð (¨Ñ*Ø& qÑ)Ø"6Ø$:Ø $× 5Ñ 5°nÓ EñˆFð +¨1Ñ-Ø%9Ø'=ñˆGñ
 Ø(5��}Ñ%Ø,9�Ð(Ñ)ØŸ/™/Ø—Y‘YØØà#Ÿ}™}ØŸ9™9Ø'2ñð
 #ð *ó 
ˆKð Ð÷[ ð ús   Ã AF!Æ!F*c                 ó¸   — |j                  d| j                  d   «      | _        |j                  d| j                  d   «      | _        |j                  d«      }|S )Nr   r   r&   )r+   r   r   r   )r    r   s     r"   rJ   zAdadelta._update_param_groupñ   sP   € Ø"Ÿ™ y°$×2DÑ2DÀYÑ2OÓPˆŒØ—N‘N 5¨$×*<Ñ*<¸UÑ*CÓDˆŒ	Ø—^‘^ HÓ-ˆ
ØÐr#   )gü©ñÒMbP?g�íµ ÷Æ°>gffffffî?NNNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   r2   r   r9   rU   rJ   Ú__classcell__)r!   s   @r"   r   r      sF   ø„ ñOðb !4ÐØ"7Ðð ØØØØØØõ
òB9ò:AöFr#   r   )r4   Úpaddler   Úbaser   Úbase.dygraphr   r   Ú	optimizerr	   Ú__all__r   © r#   r"   Ú<module>rb      s,   ðó å å Ý "Ý 'Ý  à
€ôZˆyõ Zr#   