\;jddlZddlZddlmZddlZddlmZddlmZddl m Z m Z m Z ddl mZddlmZddlmZmZmZddlmZmZmZmZdd lmZgZed#d Zed#d Zed#d Zd a dZ!dZ"dZ#GddZ$Gdde$Z%dZ&GddZ'Gdde'Z(Gdde'Z)d a*dZ+d a,dZ-Gdde'Z.e j^d$d Z0d!Z1d"Z2e'Z3e(Z4e)Z5e.Z6y)%N)NotSupportedError)_C_ops)core framework unique_name)check_variable_and_dtype)DataType)Variable check_typedefault_main_program) LayerHelperin_dynamic_modein_dynamic_or_pir_mode in_pir_mode) templatedocctrtj||Std it }t |dgddt |dtd|0tjdj|jdg}|j|j||jd}|jdd|id|id |i |S) a@ ${comment} Args: x(${x_type}): ${x_comment} max_norm(${max_norm_type}): ${max_norm_comment} name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. Returns: Tensor: out(${out_type}): ${out_comment} Examples: .. code-block:: python >>> import paddle >>> from paddle.nn import clip >>> input = paddle.to_tensor([[2.0, 2.0], [2.0, 2.0]], dtype='float32') >>> reward = clip.clip_by_norm(x=input, max_norm=1.0) >>> print(reward) Tensor(shape=[2, 2], dtype=float32, place=Place(cpu), stop_gradient=True, [[0.50000000, 0.50000000], [0.50000000, 0.50000000]]) clip_by_normX)float16float32uint16max_norm.tmpF)typenamedtype persistableOutrinputsattrsoutputs)r)rrrr localsrr floatrgenerate_with_ignorable_keyjoinrcreate_variablerr append_op)xrrhelperouts WG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/nn/clip.pyrr%sB""1h//  468 4F 30.xen= |66 HHfkk5) *   VV$agg5 ! C  Qx8$  Jctrtj|Stdit }|j |j }|jdd|iid|i|S)aQ ${comment} Args: x(${x_type}): ${x_comment} name(basestring|None): Name of the output. Returns: out(${out_type}): ${out_comment} Examples: .. code-block:: python >>> import paddle >>> import paddle.base as base >>> b = paddle.static.default_main_program().global_block() >>> var = b.create_var( ... name="X", dtype="float32", persistable=True, ... type=base.core.VarDesc.VarType.SELECTED_ROWS) >>> y = paddle.nn.clip.merge_selected_rows(var) merge_selected_rowsrrrr )r0)rrr0r r$"create_variable_for_type_inferencerr)r*rr+r,s r-r0r0bso2))!,,  ;&( ;F  3 3!'' 3 BC  "Qx  Jr.cptrtj|St|dtd|j t jjjk7r tdtdit}|j|j}|jdd|id|ii|S) a? Get tensor data from input with SelectedRows type, and outputs a Tensor. .. code-block:: text input x is SelectedRows: x.rows = [0, 5, 5, 4, 19] x.height = 20 x.value = [[1, 1] [2, 2] [2, 2] [3, 3] [6, 6]] Output is LoDTensor: out.shape = [5, 2] out.data = [[1, 1], [2, 2], [2, 2], [3, 3], [6, 6]] Args: x(SelectedRows): Input with SelectedRows type. The data type is float32, float64, int32 or int64. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name` . Returns: Variable: LoDTensor transformed from SelectedRows. The data type is same with input. Examples: .. code-block:: python >>> import paddle >>> import paddle.base as base >>> from paddle.base import core >>> paddle.enable_static() >>> scope = core.Scope() >>> block = paddle.static.default_main_program().global_block() >>> x_rows = [0, 5, 5, 4, 19] >>> height = 20 >>> x = scope.var('X').get_selected_rows() >>> x.set_rows(x_rows) >>> x.set_height(height) >>> x = block.create_var(name="X", dtype="float32", persistable=True, type=base.core.VarDesc.VarType.SELECTED_ROWS) >>> z = paddle.nn.clip.get_tensor_from_selected_rows(x) r*get_tensor_from_selected_rowszGThe type of 'x' in get_tensor_from_selected_rows must be SELECTED_ROWS.r1rrrr!r#r")r5)rrr5r r rrVarDescVarType SELECTED_ROWS TypeErrorr r$r2rr)r3s r-r5r5sZ}33A66q#x!@Avv%%333 U   EFH EF  3 3!'' 3 BC  ,Qx   Jr.Fct|dksJt|dk(r"t|dtsJt}|da|StS)Nr)len isinstancebool'_clip_by_global_norm_using_mp_type_flagargs old_values r-"_clip_by_global_norm_using_mp_typerDsL t9>> 4yA~$q'4(((; 26q'/66r.c |jtjjjk(s1|jtjjj k(r=t r3|jtjjjS|jtjk(s|jtjk(r)t r|jtjS|SN) rrr7r8FP16BF16rDastypeFP32r FLOAT16BFLOAT16r*s r-_cast_to_mp_type_if_enabledrNs 4<<'',,, 77dll**// / , .xx ,,1122 8###qww(2C2C'C , .xx &&r.ct|}trtj|Sd}t |dgd|t |fit }|j|j}d|i}d|i}|j||||S)z1 Return the squared L2 norm of a tensor. squared_l2_normr*)rfloat64rrrrrr!r#) rNrrrPrr r$r2rr))r*op_typer+r,r!r#s r-_squared_l2_normrTs $A&A%%a((G 3;W -FH -F  3 3AGG >> import paddle >>> paddle.enable_static() >>> BATCH_SIZE = 128 >>> CLIP_MAX = 2e-6 >>> CLIP_MIN = -1e-6 >>> prog = paddle.static.Program() >>> with paddle.static.program_guard(main_program=prog): ... image = paddle.static.data(name='x', shape=[None, 784], dtype='float32') ... hidden1 = paddle.static.nn.fc(image, size=128, activation='relu') ... hidden2 = paddle.static.nn.fc(hidden1, size=64, activation='relu') ... predict = paddle.static.nn.fc(hidden2, size=10, activation='softmax') ... label = paddle.static.data(name='y', shape=[1], dtype='int64') ... cost = paddle.nn.functional.cross_entropy(input=predict, label=label) ... avg_cost = paddle.mean(cost) >>> prog_clip = prog.clone() >>> prog_clip.block(0).var(hidden1.name)._set_error_clip( ... paddle.nn.clip.ErrorClipByValue( ... max=CLIP_MAX, min=CLIP_MIN)) NcXt|}|| }n t|}||_||_yrF)r%maxmin)r[rirjs r-__init__zErrorClipByValue.__init__,s.Cj ;$C*Cr.c>d|jdd|jdS)Nz ByValue, min=f, max=rjrirZs r-r\zErrorClipByValue.__str__5s!txxl&! ==r.c|jj}|jd|jd|g|j d|g|j d|j |j d|jy)Ncliprrrjri)descr)set_type set_input set_output _set_attrrjri)r[r_r` clip_op_descs r-raz ErrorClipByValue._append_clip_op8snzz++- f%sYK0 {3udhh/udhh/r.rF)rbrcrd__doc__rkr\rarer.r-rgrgs$L>0r.rgct|}|jj|jjdz }|jDcgc] }||vs| c}D]U}|j ||}t |dd}|t |ts td|D|j||Wycc}w)Nr< error_clipzIVariable's error_clip should be an instance of BaseErrorClipAttr or None.) rropop_sizeoutput_arg_names_var_recursivegetattrr>rVr:ra)r_context grad_to_varop_descngrad_nfwd_varrzs r-error_clip_callbackrAsKjjmmEJJ..0145G%668M8A>,/ /$112D9EMM> %$$\2 2r.ctrFrXr[rparamgrads r-_process_contextzClipGradBase._process_contextsr]r.ctrFrX)r[rrs r-_create_operatorszClipGradBase._create_operatorsvr]r.)rbrcrdrkr\imperative_baseno_gradrrrrrr __classcell__rs@r-rrRsE$_""""3 $$r.rcjeZdZdZdfd ZdZejdZdZ dZ dZ xZ S) ClipGradByValuea Limit the value of multi-dimensional Tensor :math:`X` to the range [min, max]. - Any values less than min are set to ``min``. - Any values greater than max are set to ``max``. The multi-dimensional Tensor :math:`X` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). Note: ``need_clip`` of ``ClipGradByValue`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. Args: max (float): The maximum value to clip by. min (float, optional): The minimum value to clip by. if not set by user, it will be set to ``-max`` automatically. In this case, ``max`` must be greater than :math:`0`. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByValue(min=-1, max=1) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() czt|| |dkDsJ| }t||_t||_y)Ng)rrkr%rirj)r[rirjrs r-rkzClipGradByValue.__init__s<  ;9 9$C::r.c>d|jdd|jdS)NzClip Gradient By Value, min = rmrnrorZs r-r\zClipGradByValue.__str__s!/|6$((1NNr.cg}|D]j\}}| t|dddur|j||f,tj||j|j }|j||fl|S)N need_clipTFr*rjri)rappendpaddlerqrjrir[rparams_and_gradsrrnew_grads r-rzClipGradByValue._dygraph_clipsx DAqyq+t,5 ''A/{{QDHH$((CH  # #QM 2! r.cg}i}tjd5|D]\}}| t|dddur|j||f,|jj j ||g5tj||j|j}ddd|j|f|j||j< dddt|||S#1swYNxYw#1swY#xYw)N gradient_cliprTFr) r name_scoperrr_program_optimized_guardrrqrjrir_correct_clip_op_role_varr[rrparam_new_grad_name_dictrrrs r-rzClipGradByValue._static_clips#%  ! !/ 2$191k40E9$++QF3WW__55q!f=%{{QDHH$((KH> ''H 63;==(0%3 ""24LM >=3 2s$AC60-C*6C6*C3 /C66C?cyrFrers r-rz ClipGradByValue._process_context r.cbtj||j|j}||fS)Nr)rrqrjrir[rrrs r-rz!ClipGradByValue._create_operatorss&;;488Bhr.rF rbrcrdrxrkr\rrrrrrrrs@r-rrzsB%NO_     $ r.rcheZdZdZfdZdZejdZdZ dZ dZ xZ S)ClipGradByNorma Limit the l2 norm of multi-dimensional Tensor :math:`X` to ``clip_norm`` . - If the l2 norm of :math:`X` is greater than ``clip_norm`` , :math:`X` will be compressed by a ratio. - If the l2 norm of :math:`X` is less than or equal to ``clip_norm`` , nothing will be done. The multidimensional Tensor :math:`X` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). The clipping formula is: .. math:: Out = \left\{ \begin{array}{ccl} X & & if (norm(X) \leq clip\_norm) \\ \frac{clip\_norm*X}{norm(X)} & & if (norm(X) > clip\_norm) \\ \end{array} \right. where :math:`norm(X)` represents the L2 norm of :math:`X`. .. math:: norm(X) = ( \sum_{i=1}^{n}|x\_i|^2)^{ \frac{1}{2}} Note: ``need_clip`` of ``ClipGradByNorm`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. Args: clip_norm(float): The maximum norm value. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByNorm(clip_norm=1.0) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() cBt|t||_yrF)rrkr% clip_norm)r[rrs r-rkzClipGradByNorm.__init__ s y)r.c d|jzS)Nz#Gradient Clip By Norm, clip_norm=%frrZs r-r\zClipGradByNorm.__str__s4t~~EEr.cg}|D]U\}}| t|dddur|j||f,t||j}|j||fW|S)NrTFr*r)rrrrrs r-rzClipGradByNorm._dygraph_clipsn DAqyq+t,5 ''A/#a$..AH  # #QM 2! r.cg}tjd5i}|D]\}}| t|dddur|j||f,|jj j ||g5t||j}dddj||j<|j||f dddt||S#1swYNxYw#1swY#xYw)NrrTFr) rrrrr_rrrrrrrs r-rzClipGradByNorm._static_clips  ! !/ 2') $$191k40E9$++QF3WW__55q!f=+a$..IH>3;==(0 ''H 6%3 ""24LM >=3 2s$AC!0C6C!C C!!C*cyrFrers r-rzClipGradByNorm._process_context1rr.c8t||j}||fS)Nr)rrrs r-rz ClipGradByNorm._create_operators4s$@hr.rrs@r-rrsB4l*F_     $ r.rct|dk(rtSt|dk(rt|dtsJt}|da|SNrr<)r=&_allow_pure_fp16_global_norm_clip_flagr>r?rAs r-!_allow_pure_fp16_global_norm_clipr<F 4yA~554yA~*T!Wd";;;: 15a.r.ct|dk(rtSt|dk(rt|dtsJt}|da|Sr)r=&_allow_pure_bf16_global_norm_clip_flagr>r?rAs r-!_allow_pure_bf16_global_norm_cliprJrr.creZdZdZ d fd ZdZejdZdZ dZ dZ dZ xZ S) ClipGradByGlobalNormao Given a list of Tensor :math:`t\_list` , calculate the global norm for the elements of all tensors in :math:`t\_list` , and limit it to ``clip_norm`` . - If the global norm is greater than ``clip_norm`` , all elements of :math:`t\_list` will be compressed by a ratio. - If the global norm is less than or equal to ``clip_norm`` , nothing will be done. The list of Tensor :math:`t\_list` is not passed from this class, but the gradients of all parameters set in ``optimizer``. If ``need_clip`` of specific param is ``False`` in its ``ParamAttr``, then the gradients of this param will not be clipped. Gradient clip will takes effect after being set in ``optimizer`` , see the document ``optimizer`` (for example: :ref:`api_paddle_optimizer_SGD`). The clipping formula is: .. math:: t\_list[i] = t\_list[i] * \frac{clip\_norm}{\max(global\_norm, clip\_norm)} where: .. math:: global\_norm = \sqrt{\sum_{i=0}^{N-1}(l2norm(t\_list[i]))^2} Note: ``need_clip`` of ``ClipGradyGlobalNorm`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. Args: clip_norm (float): The maximum norm value. group_name (str, optional): The group name for this clip. Default value is ``default_group``. auto_skip_clip (bool, optional): skip clipping gradient. Default value is ``False``. Examples: .. code-block:: python >>> import paddle >>> x = paddle.uniform([10, 10], min=-1.0, max=1.0, dtype='float32') >>> linear = paddle.nn.Linear(in_features=10, out_features=10, ... weight_attr=paddle.ParamAttr(need_clip=True), ... bias_attr=paddle.ParamAttr(need_clip=False)) >>> out = linear(x) >>> loss = paddle.mean(out) >>> loss.backward() >>> clip = paddle.nn.ClipGradByGlobalNorm(clip_norm=1.0) >>> sdg = paddle.optimizer.SGD(learning_rate=0.1, parameters=linear.parameters(), grad_clip=clip) >>> sdg.step() ct|t||_||_t |t sJ||_d|_yrF) rrkr%r group_namer>r?auto_skip_clip _async_add_n)r[rrrrs r-rkzClipGradByGlobalNorm.__init__sE y)$.$///, !r.c d|jzS)Nz+Gradient Clip By GlobalNorm, global_norm=%frrZs r-r\zClipGradByGlobalNorm.__str__s<OOr.c`g}g}g}g}|D]m\}}| t|dddur|}tr,|jrt|}|j }nG|j t jjjk(rt|}t|}t|} | jt jjjk(s1| jt jjjk(r|j| | jt jjj k(r|j| ]|j| pt#|t#|zt#|zdk(r|Sd} t#|dkDrdnd} g} t#|dkDr(| |} | j| j%| t#|dkDr?| |}| dk(r| j|n | j|j%| t#|dkDr| |}| j|| | } t'j(| } t'j*g| j|j,}d}|j.s/d}t'j0|t'j2| |  }n| |kDrd}t'j0|| }|D]\}}| t|dddur|j||f,|r`j|jk7r|j%|jn|}t'j4||}|j||f|j||f|S) NrTFrcHtj|jSrFrstacksumvar_lists r- async_add_nz7ClipGradByGlobalNorm._dygraph_clip..async_add_n<<)--/ /r.rQrshaper fill_valuer*y)rris_selected_rowsr0_get_tensor_from_selected_rowsrrr7r8r9r5rTrrGrHrrJr=rIrsqrtfullrrdividemaximummultiplyr[rrsum_square_listsum_square_list_fp16sum_square_list_fp32rr merge_grad sum_squarer sum_dtypeglobal_norm_varglobal_norm_var_fp16global_norm_var_fp32global_norm_var_fp64max_global_normrclip_var clip_inputrs r-rz"ClipGradByGlobalNorm._dygraph_clips_!! DAqyq+t,5J Q%7%7%903 'FFH 4<<//===03 ::F )*5J  DLL$8$8$=$==##t||';';'@'@@$++J7!!T\\%9%9%>%>>$++J7&&z21!8  &' (&' (    0"%_!5!9Iy  # $q (#./C#D  " "#7#>#>y#I J # $q (#./C#D I%&&';<&&';'B'B9'MN  ! ##.#?  " "#7 8%o6 ++o6 ++O11dnn  ""I}}!..?oFH .I}}/JH DAqyq+t,5 ''A/ ~~0OOAGG,! "??1j9 ''H 6 ''A/!!$ r.cFg}g}g}g}|D]\}}| t|dddur|}tr&|jrt|}t |}t |} | j tjk(s| j tjk(r|j| | j tjk(r|j| |j| t|t|zt|zdk(r|Sd} t|dkDrdnd} g} t|dkDr(| |} | j| j| t|dkDr?| |}| dk(r| j|n | j|j| t|dkDr| |}| j|| | } tj| } tj g| j |j"}d}|j$s/d}tj&|tj(| |  }n| |kDrd}tj&|| }|D]\}}| t|dddur|j||f,|r`j |j k7r|j|j n|}tj*||}|j||f|j||f|S) NrTFrcHtj|jSrFrrs r-rz3ClipGradByGlobalNorm._pir_clip..async_add_n rr.rQrrr)rris_selected_row_typer0r5rTrr rKrLrFLOAT32r=rIrrrrrrrrrs r-rzClipGradByGlobalNorm._pir_clips !! DAqyq+t,5J}!7!7!903 ::F )*5J  H$4$44##x'8'88$++J7!!X%5%55$++J7&&z2)!0  &' (&' (    0"%_!5!9Iy  # $q (#./C#D  " "#7#>#>y#I J # $q (#./C#D I%&&';<&&';'B'B9'MN  ! ##.#?  " "#7 8%o6 ++o6 ++O11dnn  ""I}}!..?oFH .I}}/JH DAqyq+t,5 ''A/ ~~0OOAGG,! "??1j9 ''H 6 ''A/!!$ r.c  g}g}g}g}g}fd}tjd5|D]w\}} |  t|dddur| } |jjj || g5| j tjjjk(rt| } t| } t| } | jtjjjk(r|j!| n| jtjjj"k(r|j!| nT| jtjjj$k(r|j!| n|j!| dddzt'|dkDrt'|dkDr t)dt'|t'|zt'|zdk(r1t'|t'|zt'|zdk(r |cdddSjjj | g5t'|dkDrdnd } g} t'|dkDrH||}|s |s t+s!| j!|j-| n| j!|t'|dkDrH||}|s |s t/s!| j!|j-| n| j!|t'|dkDr?||}| d k(r| j!|n | j!|j-| t'|dkDr||}| j!|t'| d kDr|| n| d} t1j2| } t1j4d g| jj6 }t1j8|t1j:||  }dddi}|D]\}} |  t|dddur|j!|| f-|jjj || g5t=| }|jtjjjk(rCjtjjjk7r|j-d}nv|jtjjj"k(rCjtjjj"k7r|j-d}n}t?jA}|jCd||dd|i|| ur0|jCdd|id| i|j| jdddd| jD||jD<|j!|| f dddtG||S#1swYxYw#1swYxYw#1swYixYw#1swY=xYw)Ncjr#tj|jStj|SrF)rrrradd_n)rr[s r-_add_nz1ClipGradByGlobalNorm._static_clip.._add_n^s3  ||H-1133||H--r.rrTFrz1FP16 and BF16 are not supported at the same time.rQrr<rMrrrbfloat16elementwise_mulrYrrRcastr)in_dtype out_dtyper6)$rrrr_rrrrr7r8r9r0r5rTrrGrrHrJr=rrrIrrrrrrrrNr current_blockr)rr)r[rrrrsum_square_list_bf16rrrrrrrrrglobal_norm_var_bf16rglobal_norm_var_other_dtyper scale_varrnew_g scale_inputr_s` r-rz!ClipGradByGlobalNorm._static_clipWs!!! .  ! !/ 2$191k40E9 WW__55q!f=vv!5!5!C!CC%8%; %B:%N !1*!=J!''4<<+?+?+D+DD,33J?#))T\\-A-A-F-FF,33J?#))T\\-A-A-F-FF,33J?'..z:>= %('(1,5I1JQ1N'G O$*+,*+, O$*+,*+, $M3 2P111a&9),_)=)AIy "$+,q0+12F+G(,*@B'..077 B(../CD+,q0+12F+G(,*@B'..077 B(../CD+,q0+12F+G( I-'../CD'..077 B'!+282I/#**+FG?+a/?+(+ #)++"@"(++#)//#~~# #MM%nn/J i:p(* $$191k40E9$++QF3WW__55q!f=7:E t||';';'@'@@%OOt||/C/C/H/HH&/&6&6y&A  t||';';'@'@@%OOt||/C/C/H/HH&/&6&6z&B &/ 12@@BEOO.%*=!&$ A~!'$'<%*AJ,1KK-.WW# (5>H4566(0 ''A/Y%C3^ ""24LMS>=B:9@>=Q3 2s_AX7,D-XA?X7"'X7 F?XA X7(EX+7X7X X7X( #X7+X4 0X77Yc|j|vrig||j<|j||jdz<tjdg|j|j||jdz<n*|j||jdzk(s t d|}|j tjjjk(rt|}t|}n0tr&|jrt|}t|}t|}||jj!|||_y)N _clip_valuer<r_clipz>All parameters' 'clip_norm' of a same group should be the same)rrrrr ValueErrorrrr7r8r9r0r5rrrTrr)r[rrrrlocal_norm_vars r-rz%ClipGradByGlobalNorm._process_contexts ??' )')GDOO $7;~~GDOOm3 417c2GDOOg- .>>WT__}-L%MM T 99 ,,:: :,T2J6zBJ ]t88:,T2J6zBJ)*5 ''7 r.cFd}|jdz}||jvr||j|j}tj|}|j|jdz}tj|tj ||}|j dk(sJ||j|<tr'tj||j|}||fS|jjd||j|dd |i ||fS) NcHtj|jSrFrrs r-rz;ClipGradByGlobalNorm._create_operators..async_add_nrr._scalerMrr)r<rrrrR) rrrrrrrrrr_r))r[rrrgroup_scale_namegroup_norm_varrgroup_scale_vars r-rz&ClipGradByGlobalNorm._create_operatorss 0 ??X5 4<< /(doo)FGN#[[>:N||DOOg$=>H$mm..8~>O#((D0 00->> import paddle >>> paddle.enable_static() >>> def network(): ... image = paddle.static.data(name='image', shape=[ ... None, 28], dtype='float32') ... param_attr1 = paddle.ParamAttr("fc1_param") ... fc1 = paddle.static.nn.fc(image, size=10, weight_attr=param_attr1) ... param_attr2 = paddle.ParamAttr("fc2_param") ... fc2 = paddle.static.nn.fc(fc1, size=10, weight_attr=param_attr2) ... loss = paddle.mean(fc2) ... return loss >>> # network 1: clip all parameter gradient >>> with paddle.static.program_guard(paddle.static.Program(), paddle.static.Program()): ... loss = network() ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByGlobalNorm(clip_norm=2.0)) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 2: clip parameter gradient by name >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByValue(min=-1.0, max=1.0), ... param_list=["fc1_param", "fc2_param"]) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 3: clip parameter gradient by value >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... param_var1 = paddle.static.default_main_program().global_block().var("fc1_param") ... param_var2 = paddle.static.default_main_program().global_block().var("fc2_param") ... paddle.nn.clip.set_gradient_clip( ... paddle.nn.ClipGradByValue(min=-1.0, max=1.0), ... param_list=[param_var1, param_var2]) ... sgd = paddle.optimizer.SGD(learning_rate=1e-3) ... sgd.minimize(loss) >>> # network 4: use 'set_gradient_clip' and 'optimize(grad_clip=clip)' together >>> with paddle.static.program_guard(base.Program(), paddle.static.Program()): ... loss = network() ... clip1 = paddle.nn.ClipGradByValue(min=-1.0, max=1.0) ... clip2 = paddle.nn.ClipGradByNorm(clip_norm=1.0) ... # Set the gradient clipping strategy: clip1 ... paddle.nn.clip.set_gradient_clip(clip1) ... # Set the gradient clipping strategy: clip2 ... sgd = paddle.optimizer.SGD(learning_rate=1e-3, grad_clip=clip2) ... sgd.minimize(loss) ... # 'set_gradient_clip' will not take effect when setting has a conflict, ... # and the gradient clipping strategy will be 'clip2' zCaution! 'set_gradient_clip' is not recommended and may be deprecated in future! We recommend a new strategy: set 'grad_clip' when initializing the 'optimizer'. This method can reduce the mistakes, please refer to documention of 'optimizer'.z<'clip' should be an instance of ClipGradBase's derived classNr op_namescope optimizerz'minimize' has been invoked before, this will make 'set_gradient_clip' be ineffective! Please invoke 'set_gradient_clip' before 'minimize'.c3<K|]}t|tywrF)r>str.0elems r- z$set_gradient_clip..s 8ZT:dC Zsc3PK|]}t|tj ywrF)r>r Parameterr%s r-r(z$set_gradient_clip..sLz$ 3 34s$&zK'param_list' should be a list of Parameter or basestring(parameter's name).)rrr>rr:rr r_ops all_attrsattrall_parametersallvarcopydeepcopyr)rq param_listrr{r'rs r-set_gradient_clipr4.s3t MM / dL ) J  002mmA"" R\\^ + rww @ 1  MMW  #]]1%446  8Z 88=GHZTgmmA&**40Z H LL L Y  #'==#6  Is'Ec$i}|D]\}}| |jjj||g5tjd5t |dd}||cdddcdddcSt |ts td|j|||ddddddg}i}|D]\}}| |jjj||g5tjd5j||\}}|j||j<|j||gddddddt|||S#1swYxYw#1swYrxYw#1swY:xYw#1swYxYw)Nrrz8clip attribute should be an instance of GradientClipBase)rrr)rr)r_rrrrrr>rr:rrrrr) param_gradsrrr clip_attrresrrrs r-append_gradient_clip_opsr9s|G1 9  WW__ - -q!f 5y7K7K 8  #7>I " 8 8 5 5 i6N  & &waa & H8 5 5 C!1 9  WW__ - -q!f 5y7K7K 8 (999JOE83;== $UZZ 0 JJx( ) 8 5 5c#;< J58 8 5 5$8 8 5 5sTE- E! E-1/E! E-(F>AE:F!E* &E--E7 :F ?FF cg}t|dk(ry|D]\}}| |jj}||vr$|j||jjj j D]l}|jdsd|jdvs)|jds;|jdd}||vsT|||g}|jd|ny)Nrr!r op_role_var) r=r_idxrr global_blockr+has_attrr-rv) rr block_id_listrrblock_idr{ param_name correct_p_gs r-rrsM #$)# t < ;;?? } $ X&++%%22488B N+#rww~'>>GGM*WW]3A6 !99"0<#KLL <9$r.rF)NN)7r1rsqlite3rrpaddle.autogradautogradrr paddle.baserrrpaddle.base.data_feederrpaddle.base.libpaddler paddle.common_ops_importr r r paddle.frameworkr rrr&paddle.tensor.layer_function_generatorr__all__rr0r5r@rDrNrTrVrgrrrrrrrrrdygraph_not_supportr4r9rGradientClipBaseGradientClipByValueGradientClipByNormGradientClipByGlobalNormrer.r-rRsL % )44<*OO ?  99x ##L <<~+0' 7 .$$90(90x6"%$%$PWlWtb\bJ*/&*/&V<Vr }7}7@J=4 %#/r.