Ë
    Ž\;jÖA  ã                   óz   — d dl mZ d dlmZ d dlZd dlmZ d dlmZ ddl	m
Z
  edd	d¬
«       G d„ de«      «       Zy)é    )Údefaultdict)ÚreduceN)Ú	Optimizer)Ú
deprecatedé   )Ú_strong_wolfez2.5.0zpaddle.optimizer.LBFGS)ÚsinceÚ	update_toÚlevelc                   ój   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
d	„ Zd
„ Zˆ xZS )ÚLBFGSa  
    The L-BFGS is a quasi-Newton method for solving an unconstrained optimization problem over a differentiable function.
    Closely related is the Newton method for minimization. Consider the iterate update formula:

    .. math::
        x_{k+1} = x_{k} + H_k \nabla{f_k}

    If :math:`H_k` is the inverse Hessian of :math:`f` at :math:`x_k`, then it's the Newton method.
    If :math:`H_k` is symmetric and positive definite, used as an approximation of the inverse Hessian, then
    it's a quasi-Newton. In practice, the approximated Hessians are obtained
    by only using the gradients, over either whole or part of the search
    history, the former is BFGS, the latter is L-BFGS.

    Reference:
        Jorge Nocedal, Stephen J. Wright, Numerical Optimization, Second Edition, 2006. pp179: Algorithm 7.5 (L-BFGS).

    Args:
        learning_rate (float, optional): learning rate .The default value is 1.
        max_iter (int, optional): maximal number of iterations per optimization step.
            The default value is 20.
        max_eval (int, optional): maximal number of function evaluations per optimization
            step. The default value is max_iter * 1.25.
        tolerance_grad (float, optional): termination tolerance on first order optimality
            The default value is 1e-5.
        tolerance_change (float, optional): termination tolerance on function
            value/parameter changes. The default value is 1e-9.
        history_size (int, optional): update history size. The default value is 100.
        line_search_fn (string, optional): either 'strong_wolfe' or None. The default value is strong_wolfe.
        parameters (list|tuple, optional): List/Tuple of ``Tensor`` names to update to minimize ``loss``. \
            This parameter is required in dygraph mode. The default value is None.
        weight_decay (float|WeightDecayRegularizer, optional): The strategy of regularization. \
            It canbe a float value as coeff of L2 regularization or \
            :ref:`api_paddle_regularizer_L1Decay`, :ref:`api_paddle_regularizer_L2Decay`.
            If a parameter has set regularizer using :ref:`api_paddle_ParamAttr` already, \
            the regularization setting here in optimizer will be ignored for this parameter. \
            Otherwise, the regularization setting here in optimizer will take effect. \
            Default None, meaning there is no regularization.
        grad_clip (GradientClipBase, optional): Gradient cliping strategy, it's an instance of \
            some derived class of ``GradientClipBase`` . There are three cliping strategies \
            ( :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , \
            :ref:`api_paddle_nn_ClipGradByValue` ). Default None, meaning there is no gradient clipping.
        name (str, optional): Normally there is no need for user to set this property.
            For more information, please refer to :ref:`api_guide_Name`.
            The default value is None.

    Return:
        loss (Tensor): the final loss of closure.

    Examples:
        .. code-block:: python

            >>> import paddle
            >>> import numpy as np
            >>> from paddle.incubate.optimizer import LBFGS

            >>> paddle.disable_static()
            >>> np.random.seed(0)
            >>> np_w = np.random.rand(1).astype(np.float32)
            >>> np_x = np.random.rand(1).astype(np.float32)

            >>> inputs = [np.random.rand(1).astype(np.float32) for i in range(10)]
            >>> # y = 2x
            >>> targets = [2 * x for x in inputs]

            >>> class Net(paddle.nn.Layer):
            ...     def __init__(self):
            ...         super().__init__()
            ...         w = paddle.to_tensor(np_w)
            ...         self.w = paddle.create_parameter(shape=w.shape, dtype=w.dtype, default_initializer=paddle.nn.initializer.Assign(w))
            ...     def forward(self, x):
            ...         return self.w * x

            >>> net = Net()
            >>> opt = LBFGS(learning_rate=1, max_iter=1, max_eval=None, tolerance_grad=1e-07, tolerance_change=1e-09, history_size=100, line_search_fn='strong_wolfe', parameters=net.parameters())
            >>> def train_step(inputs, targets):
            ...     def closure():
            ...         outputs = net(inputs)
            ...         loss = paddle.nn.functional.mse_loss(outputs, targets)
            ...         print('loss: ', loss.item())
            ...         opt.clear_grad()
            ...         loss.backward()
            ...         return loss
            ...     opt.step(closure)

            >>> for input, target in zip(inputs, targets):
            ...     input = paddle.to_tensor(input)
            ...     target = paddle.to_tensor(target)
            ...     train_step(input, target)

    c                 óö  •— |€|dz  dz  }|| _         || _        || _        || _        || _        || _        || _        t        |t        j                  «      rt        dt        |«      z   «      ‚t        t        «      | _        t        ‰| �A  d||	|
|¬«       t        | j"                  d   t        «      s| j"                  | _        d | _        y t'        | j(                  «      D ]  \  }}|d   | _        Œ d | _        y )Né   é   z^parameters argument given to the optimizer should be an iterable of Tensors or dicts, but got ç      ð?)Úlearning_rateÚ
parametersÚweight_decayÚ	grad_clipÚnamer   Úparams)r   Úmax_iterÚmax_evalÚtolerance_gradÚtolerance_changeÚhistory_sizeÚline_search_fnÚ
isinstanceÚpaddleÚTensorÚ	TypeErrorÚtyper   ÚdictÚstateÚsuperÚ__init__Ú_parameter_listÚ_paramsÚ	enumerateÚ_param_groupsÚ_numel_cache)Úselfr   r   r   r   r   r   r   r   r   r   r   ÚidxÚparam_groupÚ	__class__s                 €úhG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/incubate/optimizer/lbfgs.pyr&   zLBFGS.__init__w   s  ø€ ð ÐØ !‘| qÑ(ˆHà*ˆÔØ ˆŒØ ˆŒØ,ˆÔØ 0ˆÔØ(ˆÔØ,ˆÔä�j¤&§-¡-Ô0Üð<Ü>BÀ:Ó>NñOóð ô
 !¤Ó&ˆŒ
ä‰ÑØØ!Ø%ØØð 	ô 	
ô ˜$×.Ñ.¨qÑ1´4Ô8Ø×/Ñ/ˆDŒLð
 !ˆÕô %.¨d×.@Ñ.@Ö$AÑ ��[Ø*¨8Ñ4�•ð %Bð !ˆÕó    c                 óx   — i }| j                   j                  «       D ]  \  }}|j                  ||i«       Œ d|iS )zÍReturns the state of the optimizer as a :class:`dict`.

        Return:
            state, a dict holding current optimization state. Its content
                differs between optimizer classes.
        r$   )r$   ÚitemsÚupdate)r,   Úpacked_stateÚkÚvs       r0   Ú
state_dictzLBFGS.state_dict¨   sC   € ð ˆØ—J‘J×$Ñ$Ö&‰DˆAˆqØ×Ñ  A Õ'ð 'ð ˜Ð&Ð&r1   c                 ól   — | j                   €t        d„ | j                  d«      | _         | j                   S )Nc                 ó(   — | |j                  «       z   S ©N)Únumel)ÚtotalÚps     r0   Ú<lambda>zLBFGS._numel.<locals>.<lambda>º   s   €  ¨¯©«Ò!2r1   r   )r+   r   r(   )r,   s    r0   Ú_numelzLBFGS._numel¶   s4   € à×ÑÐ$Ü &Ù2°D·L±LÀ!ó!ˆDÔð × Ñ Ð r1   c                 ó  — g }| j                   D ]a  }|j                  €&t        j                  |«      j	                  dg«      }n|j                  j	                  dg«      }|j                  |«       Œc t        j                  |d¬«      S )Néÿÿÿÿr   )Úaxis)r(   Úgradr   Ú
zeros_likeÚreshapeÚappendÚconcat)r,   Úviewsr>   Úviews       r0   Ú_gather_flat_gradzLBFGS._gather_flat_grad¿   sn   € ØˆØ—”ˆAØ�v‰vˆ~Ü×(Ñ(¨Ó+×3Ñ3°R°DÓ9‘à—v‘v—~‘~ r dÓ+�Ø�L‰L˜Õð ô �}‰}˜U¨Ô+Ð+r1   c           	      ó  — d}| j                   D ]e  }t        d„ |j                  «      }t        j                  |j                  ||||z    j                  |j                  «      |z  «      |«      }||z  }Œg || j                  «       k(  sJ ‚y )Nr   c                 ó   — | |z  S r;   © )ÚxÚys     r0   r?   z!LBFGS._add_grad.<locals>.<lambda>Í   s   € ¨¨Aªr1   )r(   r   Úshaper   ÚassignÚaddrF   r@   )r,   ÚalphaÚ	directionÚoffsetr>   r<   s         r0   Ú	_add_gradzLBFGS._add_gradÊ   sˆ   € ØˆØ—”ˆAÜÑ-¨q¯w©wÓ7ˆEÜ—‘Ø—‘Ø˜f v°¡~Ð6×>Ñ>¸q¿w¹wÓGÈ%ÑOóð ó	ˆAð �e‰O‰Fð ð ˜Ÿ™›Ò&Ð&Ñ&r1   c                 ó\   — | j                   D �cg c]  }|j                  «       ‘Œ c}S c c}w r;   )r(   Úclone)r,   r>   s     r0   Ú_clone_paramzLBFGS._clone_param×   s$   € Ø#'§<¢<Ó0¡<˜a�—‘•	 <Ñ0Ð0ùÒ0s   �)c                 ól   — t        | j                  |«      D ]  \  }}t        j                  ||«       Œ y r;   )Úzipr(   r   rR   )r,   Úparams_datar>   Úpdatas       r0   Ú
_set_paramzLBFGS._set_paramÚ   s)   € Ü˜DŸL™L¨+Ö6‰HˆAˆuÜ�M‰M˜% Õ#ñ 7r1   c                 ó�   — | j                  ||«       t         |«       «      }| j                  «       }| j                  |«       ||fS r;   )rW   ÚfloatrK   r_   )r,   ÚclosurerO   rT   ÚdÚlossÚ	flat_grads          r0   Ú_directional_evaluatezLBFGS._directional_evaluateÞ   s@   € Ø�‰�u˜aÔ Ü‘W“YÓˆØ×*Ñ*Ó,ˆ	Ø�‰˜ÔØ�YˆÐr1   c           
      óÚ  ‡ ‡— t        j                  «       5   t        j                  «       ‰«      Š‰ j                  }‰ j                  }‰ j
                  }‰ j                  }‰ j                  }‰ j                  }‰ j                  }‰ j                  }	|	j                  dd«       |	j                  dd«        ‰«       }
t        |
«      }d}|	dxx   dz  cc<   ‰ j                  «       }|j                  «       j                  «       |k  }|r|
cddd«       S |	j!                  d«      }|	j!                  d«      }|	j!                  d«      }|	j!                  d	«      }|	j!                  d
«      }|	j!                  d«      }|	j!                  d«      }|	j!                  d«      }d}||k  �rý|dz  }|	dxx   dz  cc<   |	d   dk(  r9|j#                  «       }g }g }g }t        j$                  d|
j&                  ¬«      }�nã|j)                  |«      }|j+                  t        j$                  ||j&                  ¬«      «      }|j-                  |«      }|dkD  r‹t/        |«      |k(  r3|j1                  d«       |j1                  d«       |j1                  d«       |j3                  |«       |j3                  |«       |j3                  d|z  «       ||j-                  |«      z  }t/        |«      }d|	vr	dg|z  |	d<   |	d   }|j#                  «       }t5        |dz
  dd«      D ]N  }||   j-                  |«      ||   z  ||<   t        j6                  |j9                  ||   ||    z  «      |«       ŒP t        j*                  ||«      x}}t5        |«      D ]M  }||   j-                  |«      ||   z  } t        j6                  |j9                  ||   ||   | z
  z  «      |«       ŒO |€|j;                  «       }nt        j6                  ||«       |}|	d   dk(  r/t=        dd|j                  «       j?                  «       z  «      |z  }n|}|j-                  |«      }!|!| kD  r�nJd}"|�p|dk7  rtA        d«      ‚‰ jC                  «       }#ˆˆ fd„}$tE        |$|#|||||!«      \  }}}}"‰ jG                  ||«       |j                  «       j                  «       |k  }nw‰ jG                  ||«       ||k7  r`t        j                  «       5  t         ‰«       «      }ddd«       ‰ j                  «       }|j                  «       j                  «       |k  }d}"||"z  }|	dxx   |"z  cc<   |rnJ||z  j                  «       j                  «       |k  rn%t        ||z
  «      |k  rn||k\  rn||k(  rn||k  r�Œý||	d<   ||	d<   ||	d<   ||	d	<   ||	d
<   ||	d<   ||	d<   ||	d<   ddd«       |
S # 1 sw Y   ŒÍxY w# 1 sw Y   
S xY w)z±
        Performs a single optimization step.

        Args:
            closure (callable): A closure that reevaluates the model
                and returns the loss.

        Ú
func_evalsr   Ún_iterr   Nrc   rT   Úold_ykÚold_skÚroÚH_diagÚprev_flat_gradÚ	prev_lossr   )Údtypeg»½×Ùß|Û=ÚalrB   Ústrong_wolfez only 'strong_wolfe' is supportedc                 ó,   •— ‰j                  ‰| ||«      S r;   )rf   )rO   rT   rc   rb   r,   s      €€r0   Úobj_funczLBFGS.step.<locals>.obj_funcq  s   ø€ Ø#'×#=Ñ#=Ø '¨¨E°1ó$ð r1   )$r   Úno_gradÚenable_gradr   r   r   r   r   r   r   r$   Ú
setdefaultra   rK   ÚabsÚmaxÚgetÚnegÚ	to_tensorrp   ÚsubtractÚmultiplyÚdotÚlenÚpoprG   ÚrangerR   rS   rY   ÚminÚsumÚRuntimeErrorrZ   r   rW   )%r,   rb   r   r   r   r   r   r   r   r$   Ú	orig_lossrd   Úcurrent_evalsre   Úopt_condrc   rT   rj   rk   rl   rm   rn   ro   ri   rP   ÚsÚysÚnum_oldrq   ÚqÚiÚrÚbe_iÚgtdÚls_func_evalsÚx_initrt   s%   ``                                   r0   Ústepz
LBFGS.stepå   sÕ  ù€ ô �^‰^Õà*”f×(Ñ(Ó*¨7Ó3ˆGà ×.Ñ.ˆMØ—}‘}ˆHØ—}‘}ˆHØ!×0Ñ0ˆNØ#×4Ñ4ÐØ!×0Ñ0ˆNØ×,Ñ,ˆLØ—J‘JˆEØ×Ñ˜\¨1Ô-Ø×Ñ˜X qÔ)ñ  ›	ˆIÜ˜Ó#ˆDàˆMØ�,Ó 1Ñ$Óà×.Ñ.Ó0ˆIØ —}‘}“×*Ñ*Ó,°Ñ>ˆHñ Ø ÷7 Ñð< —	‘	˜#“ˆAØ—I‘I˜gÓ&ˆEØ—Y‘Y˜xÓ(ˆFØ—Y‘Y˜xÓ(ˆFØ—‘˜4“ˆBØ—Y‘Y˜xÓ(ˆFØ"ŸY™YÐ'7Ó8ˆNØŸ	™	 +Ó.ˆIàˆFà˜8Ó#à˜!‘�Ø�h“ 1Ñ$“ð
 ˜‘? aÒ'Ø!Ÿ™›�AØ�FØ�FØ�BÜ#×-Ñ-¨c¸¿¹ÔI’Fð "×*Ñ*¨>Ó:�AØŸ
™
¤6×#3Ñ#3°EÀÇÁÔ#IÓJ�AØŸ™˜q›�BØ˜E’zä˜v›;¨,Ò6à"ŸJ™J qœMØ"ŸJ™J qœMØŸF™F 1œIð Ÿ™ aÔ(ØŸ™ aÔ(ØŸ	™	 #¨¡(Ô+ð "$ a§e¡e¨A£h¡˜ô " &›k�Gà 5Ñ(Ø'+ f¨|Ñ&;˜˜d™Ø˜t™�Bð "Ÿ™›�AÜ" 7¨Q¡;°°BÖ7˜Ø & q¡	§¡¨aÓ 0°2°a±5Ñ 8˜˜1™ÜŸ™ a§e¡e¨F°1©I¸"¸Q¹%¸Ñ,@Ó&AÀ1ÕEð 8ô #ŸO™O¨A¨vÓ6Ð6�A˜Ü" 7ž^˜Ø% a™yŸ}™}¨QÓ/°"°Q±%Ñ7˜ÜŸ™ a§e¡e¨F°1©I¸¸A¹À¹Ñ,FÓ&GÈÕKð ,ð "Ð)Ø%.§_¡_Ó%6‘Nä—M‘M )¨^Ô<Ø �	ð ˜‘? aÒ'ä˜C  y§}¡}£×':Ñ':Ó'<Ñ!<Ó=ÀÑMñ ð *�Eð  —m‘m AÓ&�ð Ð*Ð*Ò*Ùð !"�Ø!Ð-à%¨Ò7Ü*Ð+MÓNÐNà!%×!2Ñ!2Ó!4˜õô
 ANØ$ f¨e°Q¸¸iÈóAÑ=˜˜i¨°ð —N‘N 5¨!Ô,Ø(Ÿ}™}›×2Ñ2Ó4¸ÑF‘Hð —N‘N 5¨!Ô,Ø Ò)Ü#×/Ñ/Õ1Ü#(©«Ó#3˜D÷ 2à$(×$:Ñ$:Ó$<˜	Ø#,§=¡=£?×#6Ñ#6Ó#8¸NÑ#J˜Ø()˜ð  Ñ.�Ø�lÓ# }Ñ4Ó#ñ Øð ˜‘I—?‘?Ó$×(Ñ(Ó*Ð.>Ò>Øä�t˜iÑ'Ó(Ð+;Ò;Øð ! HÒ,Øà˜XÒ%ØðC ˜8Ô#ðF ˆE�#‰JØ"ˆE�'‰NØ$ˆE�(‰OØ$ˆE�(‰OØˆE�$‰KØ$ˆE�(‰OØ&4ˆEÐ"Ñ#Ø!*ˆE�+Ñ÷g ðj Ð÷K 2Ð1ú÷a ðj Ðús2   —C4W ÄO"W Ó7WÔBW Ö"(W ×W	×W × W*)r   é   NgH¯¼šò×z>g•Ö&è.>éd   NNNNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r&   r8   r@   rK   rW   rZ   r_   rf   r“   Ú__classcell__)r/   s   @r0   r   r      s\   ø„ ñYðz ØØØØØØØØØØõ/!òb'ò!ò,ò'ò1ò$òör1   r   )Úcollectionsr   Ú	functoolsr   r   Úpaddle.optimizerr   Úpaddle.utilsr   Úline_search_dygraphr   r   rN   r1   r0   Ú<module>r       sB   ðõ  $Ý ã Ý &Ý #å .ñ �'Ð%=ÀQÔGôIˆIó Ió HñIr1   