\;jgIddlZddlZddlmZddlmZddlmZddl m Z m Z m Z m Z mZmZgZej"j$j&dej"j$j(d ej"j$j*d ej"j$j,d ej"j$j.d ej"j$j0d ej"j$j2dej"j$j4dej"j$j6dej"j$j8dej"j$j:dej"j$j<di Zej"j$j&ej"j$j(ej"j$j*ej"j$j,ej"j$j.ej"j$j0ej"j$j2ej"j$j4ej"j$j6ej"j$j8ej"j$j:ej"j$j<d Z ejBj&dejBjDd ejBjFd ejBjHd ejBjJd ejBj0d ejBj2dejBj4dejBj6dejBj8dejBj:dejBj<di Z&d#dZ'dZ(dZ) d$dZ*d$dZ+ d$dZ,e-e.e fe/e fdfdZ0GddZ1Gdd Z2Gd!d"Z3y)%N)Value) ParameterMeta)core)Variable_cpu_num _cuda_idsdefault_main_programin_dygraph_mode in_pir_modeboolfloat16uint16float32float64int8int16int32int64uint8 complex64 complex128 rrrrrrrrrrrrc|jdk(r|jtjS|dk(rtj|gd}tj dtjg|j }tj||j}|dk(rtj|gd}|S)NrNHWC)rrrcbtjdtjd|ddz S)Nz)convert_float_to_uint16..Ws$&--fkk$&:;A>"Dotypes)rrrr) sizeviewnpr transpose vectorizeflatreshapeshape)data data_formatnew_datas r(convert_float_to_uint16r8Os yyA~yy##f||D,/r||D { iiHzz(DJJ/Hf<<,7 Or*ctjdtjg|j}tj||j S)Ncbtjdtjd|dzdS)Nr rr!rr"r&s r(r)z)convert_uint16_to_float..cs#&--fkk$R&@A!Dr*r+)r/r1rr2r3r4)r5r7s r(convert_uint16_to_floatr;asEr||D | iiH ::h ++r*c t|tjjr|tvr t|St|tj r|t vrt |St|tr|ttjtjtjtjtjtjtj tj"tj$tj&tj(f vr |j*S|dvr t-|S|dvryt/d|z)Nr)bfloat16rzdtype must be any of [bool, float16, uint16, float32, float64, int8, int16, int32, int64, uint8, complex64, complex128, bfloat16], but received %s) isinstancerVarDescVarType_PADDLE_DTYPE_2_NUMPY_DTYPEDataType_PADDLE_PIR_DTYPE_2_NUMPY_DTYPEtyperr/rrrrrrrrrrr__name__str TypeErrordtypes r( convert_dtyperJis%--. / /.u5 5%' 3 3259 9 E4   JJ II JJ JJ GG HH HH HH HH LL MM  >> !   "u:  L   Q   r*ctrt||ttf||nt||t||t |j ||||yN)r check_typerrr check_dtyperI)input input_nameexpected_dtypeop_name extra_messages r(check_variable_and_dtyperTsG} :}5w   5*hG Z-Pr*c jtryddlm}|r2t|ts|f}|t j jfz }n6t|t j jrtd|d|dt||s'tdj|||t||y)Nr)in_to_static_modezPlease use `with base.dygraph.guard()` as context or `base.enable_dygraph()` to switch to imperative mode firstly. Because received 'z' in z is a imperative Variable.z6The type of '{}' in {} must be {}, but received {}. {}) r dygraph.baserVr>tuplereagerTensorrGformatrD)rOrP expected_typerRrSrVs r(rMrMs 0-/*,M$**++-- E4::,, - !!+ E':T V   e] + D K KG]DK    ,r*c tryt||vr'tdj|||t||y)Nz;The data type of '{}' in {} must be {}, but received {}. {})r rJrGr[) input_dtyperPrQrRrSs r(rNrNsN[!7 I P Pk*    8r*)rrc rtryt|d|||pt|ts`|D][}t|d|||t|ts%t |j d||dj dj|]|*t|trt |j d||yyy)Nr4zelement of shapez9If element of shape is Tensor, its data type should be {}z, )r rMr>rrNrIr[join)r4rRexpected_shape_typeexpected_element_typeexpected_tensor_dtypeitems r( check_shaperesug2G<(E81LD t/1F P$0Zh5OJJ&)OVV "78 (Zx-HEKK*?I.I(r*c0eZdZdZdZdZdZdZdZy)DataToLoDTensorConverterc||_||_||_d}|jD]}|dkr|dz }|dkDsd|_nt||_|j yNrr)place lod_levelr4rJrI_reset)selfrjrkr4rI negtive_countss r(__init__z!DataToLoDTensorConverter.__init__sd "  A1u" q !   #5)  r*clg|_t|jDcgc]}gc}|_ycc}wrL)r5rangerklod)rm_s r(rlzDataToLoDTensorConverter._resets- %dnn 56 51B 566s 1cR|j||j|jyrL) _feed_impl_rsrk)rmr5s r(feedzDataToLoDTensorConverter.feeds txx8r*c|dk(r|jj|y|djt||D]}|j||dd|dz yri)r5appendlenrv)rmr5rsrk each_datas r(rvz$DataToLoDTensorConverter._feed_impl_sU > II  T " FMM#d) $!   CGY]C"r*ct|j|D];\}}||k7s |dk\s|dk\stdj|j|y)NrzDShape not match. What is defined in data layer is {}, but receive {})zipr4 ValueErrorr[)rmr4s1s2s r( _check_shapez%DataToLoDTensorConverter._check_shapesR$**e,FBRxB!Ga Zaa E-r*cNtj|j|j}|jrGt |jt |jk7r |j |j}tj}|j||j|jdkDr|j|j|j!|S#t$r0tdj|j|jwxYw)NrHzBReshape error. What is defined in data layer is {}, but receive {}r)r/arrayr5rIr4rzr3r~r[r LoDTensorsetrjrkset_recursive_sequence_lengthsrsrl)rmarrts r(donezDataToLoDTensorConverter.done&shhtyy 3 ::399~TZZ0++djj1C NN  c4:: >>A  , ,TXX 6 "$\cc JJ s $C++9D$N) rE __module__ __qualname__rprlrwrvrrr*r(rgrgs! 79Dr*rgceZdZdZdZdZy)BatchedTensorProviderc ||_||_||_g|_||_|D]^}|j dk(sJd|jj t|jd|j|j`y)Nrzlod_level must be 0rjrkr4rI) rj batch_size generator converters drop_lastrkryrgr4rI)rm feed_listrjrrrvars r(rpzBatchedTensorProvider.__init__;sy $""C==A% <'< <% OO " "(**))))  r*c\|jDcgc]}|jc}Scc}wrL)rr)rmcs r(_donezBatchedTensorProvider._doneMs$"&//2/Q/222s)c#Kd}|jD]d}t||jD] \}}|jj |"|dz }||j k(sQd}|j f|js|dkDr|j y|jDcgc]}|jc}ycc}wwri) rr}rr5ryrrrrl)rmidx each_sample each_sloteach_converterrs r(__call__zBatchedTensorProvider.__call__Ps>>+K-0doo-N) >##**95.O 1HCdoo%jjl",~~#'**, !% 1AQXXZ 1 1sA#C&AC.C  CN)rErrrprrrr*r(rr:s$32r*rc$eZdZdZddZdZdZy) DataFeedera :api_attr: Static Graph DataFeeder converts the data that returned by a reader into a data structure that can feed into Executor. The reader is usually a python generator that returns a list of mini-batch data entries. Parameters: feed_list (list): Variables or names of Variables that need to feed. place (:ref:`api_paddle_CPUPlace` | :ref:`api_paddle_CUDAPlace` ): place indicates the device (CPU | GPU) the data will be fed into, if you want to feed data into GPU, please using :code:`base.CUDAPlace(i)` (:code:`i` represents the GPU id), or if you want to feed data into CPU, please using :code:`base.CPUPlace()`. program (:ref:`api_paddle_static_Program` , optional): The Program that will feed data into, if program is None, it will use default_main_program(). Default None. Raises: :code:`ValueError` - If some Variables are not in this Program. Example: .. code-block:: python >>> import numpy as np >>> import paddle >>> from paddle import base >>> paddle.enable_static() >>> place = paddle.CPUPlace() >>> def reader(): ... for _ in range(4): ... yield np.random.random([4]).astype('float32'), np.random.random([3]).astype('float32'), ... >>> main_program = paddle.static.Program() >>> startup_program = paddle.static.Program() >>> with paddle.static.program_guard(main_program, startup_program): ... data_1 = paddle.static.data(name='data_1', shape=[None, 2, 2], dtype='float32') ... data_2 = paddle.static.data(name='data_2', shape=[None, 1, 3], dtype='float32') ... out = paddle.static.nn.fc(x=[data_1, data_2], size=2) ... # ... >>> feeder = base.DataFeeder([data_1, data_2], place) >>> exe = paddle.static.Executor(place) >>> exe.run(startup_program) >>> feed_data = feeder.feed(reader()) >>> # print feed_data to view feed results >>> # print(feed_data['data_1']) >>> # print(feed_data['data_2']) >>> outs = exe.run( ... program=main_program, ... feed=feed_data, ... fetch_list=[out] ... ) >>> print(outs) Nc.g|_g|_g|_g|_| t }|D]}t |t r |jdj|}t |ts td|jj|j|jj|j|jj|j|jj|j||_y)Nrz+Feed list should contain a list of variable) feed_dtypes feed_names feed_shapesfeed_lod_levelr r>rFblockrrrGryrInamerkr4rj)rmrrjprogrameach_vars r(rpzDataFeeder.__init__s  ?*,G!H(C("==+//9h1 MNN    # #HNN 3 OO " "8== 1    & &x'9'9 :    # #HNN 3" r*c g}t|j|j|jD].\}}}|j t |j |||0|D]\}t|t|k(sJdt|t|fzt||D]\}}|j|^i} t|j|D]\} }|j| | <| S)au According to :code:`feed_list` of :code:`DataFeeder` and :code:`iterable` , converts the input into a data structure that can feed into Executor. Parameters: iterable (generator): user defined python generator to read the raw input data Returns: :code:`dict`: a :code:`dict` that contains (variable name - converted tensor) pairs Example: .. code-block:: python >>> # In this example, reader - generator will return a list of ndarray of 3 elements >>> # feed API will convert each ndarray input into a tensor >>> # the return result is a dict with keys: data_1, data_2, data_3 >>> # result['data_1'] a LoD-Tensor with shape of [5, 2, 1, 3]. 5 is batch size, and [2, 1, 3] is the real shape of data_1. >>> # result['data_2'], result['data_3'] are similar. >>> import numpy as np >>> import paddle >>> from paddle import base >>> paddle.enable_static() >>> def reader(limit=5): ... for i in range(1, limit + 1): ... yield np.ones([6]).astype('float32') * i , np.ones([1]).astype('int64') * i, np.random.random([9]).astype('float32') ... >>> data_1 = paddle.static.data(name='data_1', shape=[None, 2, 1, 3]) >>> data_2 = paddle.static.data(name='data_2', shape=[None, 1], dtype='int64') >>> data_3 = paddle.static.data(name='data_3', shape=[None, 3, 3], dtype='float32') >>> feeder = base.DataFeeder(['data_1','data_2', 'data_3'], paddle.CPUPlace()) >>> result = feeder.feed(reader()) >>> print(result['data_1']) >>> print(result['data_2']) >>> print(result['data_3']) rzDThe number of fields in data (%d) does not match len(feed_list) (%d)) r}rrrryrgrjrzrwrr) rmiterable converterrkr4rIrrrret_dict each_names r(rwzDataFeeder.feedsP '*   !1!143C3C( #Iue   (**'  ( $K{#s9~5 3([!3y>283 35.1K-H) ##I..I $),T__i)H %I~"0"5"5"7HY *Ir*c| t|St|jtjrt t StSrL)intr>rjr CUDAPlacerzr r )rm num_placess r(_get_number_of_places_z!DataFeeder._get_number_of_places_s9  !z? "  DNN 3y{# #: r*rL)rErr__doc__rprwrrr*r(rras=~&?Br*r)NCHW))4r#numpyr/pirrpir.corerrr frameworkrr r r r r __all__r?r@BOOLFP16BF16FP32FP64INT8INT16INT32INT64UINT8 COMPLEX64 COMPLEX128rA_NUMPY_DTYPE_2_PADDLE_DTYPErBFLOAT16BFLOAT16FLOAT32FLOAT64rCr8r;rJrTrMrNlistrXrrergrrrr*r(rsE$  LLvLLyLLxLLyLLyLLvLLLLLLLLLL""KLL##\  LL % %||##((ll""''||##((||##(( LL % % \\ ! ! ' ' \\ ! ! ' ' \\ ! ! ' ' \\ ! ! ' '%%//,,&&11   MMMM9MMHMM9MM9MMMMMMMMMMMM[MMl # $,6t?A Q FEG ,uh//, J888v$2$2NZZr*