\;jdZddlZddlZddlmZgZdZdZdZ dZ dZ d Z d Z d Zd Zd ZdZdZedddddZedddddZedddddZeddddddZedddddZy)a WMT14 dataset. The original WMT14 dataset is too large and a small set of data for set is provided. This module will download dataset from http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz and parse training set and test set into paddle reader creators. N) deprecatedzJhttp://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz 7d7897317ddd8ba0ae5c5fa7248d3ff5z/http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz 0791583d57d5beb693b9414c5b36798cz1http://paddlemodels.bj.bcebos.com/wmt%2Fwmt14.tgz 0cb4a5366189b6acba876491c8724fa3zzzcd}tj|d5}|Dcgc])}|jjdr |j+}}t |dk(sJ||j |d|}|Dcgc])}|jjdr |j+}}t |dk(sJ||j |d|}||fcdddScc}wcc}w#1swYyxYw)Nci}t|D].\}}||kr"|||jj<-|S|S)N) enumeratestripdecode)fdsizeout_dict line_countlines ]G:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/dataset/wmt14.py __to_dictz!__read_to_dict..__to_dict2sJ )" JD 2<,,./ !. rmodezsrc.dictrztrg.dict)tarfileopennameendswithlen extractfile)tar_file dict_sizerf each_itemnamessrc_dicttrg_dicts r__read_to_dictr&1s hS )Q  ~~&&z2 NN  5zQQ]]584i@  ~~&&z2 NN  5zQQ]]584i@! * )   * )s(C1.C'1C1?.C,-0C1' C11C:cfd}|S)Nc 3DKt\}}tjd5}|Dcgc])}|jj r |j+}}|D]}|j |D]}|j }|jjd}t|dk7rA|d}|j} tg| ztgzD cgc]} |j| t} } |d} | j} | D cgc]} |j| t}} t| dkDst|dkDr||tgz}|tg|z}| ||f dddycc}wcc} wcc} w#1swYyxYww)Nrr rrrP)r&rrrrrr r splitrSTARTENDgetUNK_IDX)r$r%r!r"r#rr line_splitsrc_seq src_wordswsrc_idstrg_seq trg_wordstrg_ids trg_ids_nextr file_namers rreaderzreader_creator..readerNs+Hi@( \\( -"#!"I>>**95!"  MM$/D;;=D!%!3!3D!9J:!+ (mG ' I#(9!4u!<!Please use new dataset API which supports paddle.io.DataLoader)since update_tolevelreasonc~ttjjj t dt d|S)a WMT14 training set creator. It returns a reader creator, each sample in the reader is source language word ID sequence, target language word ID sequence and next word ID sequence. :return: Training reader creator :rtype: callable wmt14z train/trainr<paddledatasetcommondownload URL_TRAIN MD5_TRAINr s rtrainrKrs3" &&y'9E rc~ttjjj t dt d|S)z WMT14 test set creator. It returns a reader creator, each sample in the reader is source language word ID sequence, target language word ID sequence and next word ID sequence. :return: Test reader creator :rtype: callable rBz test/testrCrJs rtestrMs3" &&y'9E rc~ttjjj t dt d|S)NrBzgen/genrCrJs rgenrOs3 &&y'9E rc6tjjjtdt }t ||\}}|rF|jDcic]\}}|| }}}|jDcic]\}}|| }}}||fScc}}wcc}}wNrB)rDrErFrGrHrIr&items)r reverserr$r%kvs rget_dictrVs~~$$--i)LH')<Hh%-^^%56%5TQAqD%56%-^^%56%5TQAqD%56 X 76s  B; Bctjjjtdt tjjjt dtyrQ)rDrErFrGrHrI URL_MODEL MD5_MODELr;rrfetchrZs< NN""9gyA NN""9gyAr)T)__doc__rpaddle.dataset.commonrD paddle.utilsr__all__ URL_DEV_TEST MD5_DEV_TESTrHrIrXrYr,r-UNKr/r&r<rKrMrOrVrZr;rrrbs# ) 2  > . ? .     "8"J * K    $ * K    $ * K     * K     * K  B  Br