\;jVddlZddlZddlZddlmZddlmZgZdZ dZ GddeZ y)N)_check_exists_and_download)Datasetz>> import paddle >>> from paddle.text.datasets import Imikolov >>> class SimpleNet(paddle.nn.Layer): ... def __init__(self): ... super().__init__() ... ... def forward(self, src, trg): ... return paddle.sum(src), paddle.sum(trg) >>> imikolov = Imikolov(mode='train', data_type='SEQ', window_size=2) >>> for i in range(10): ... src, trg = imikolov[i] ... src = paddle.to_tensor(src) ... trg = paddle.to_tensor(trg) ... ... model = SimpleNet() ... src, trg = model(src, trg) ... print(src.item(), trg.item()) 2076 2075 2076 2075 675 674 4 3 464 463 2076 2075 865 864 2076 2075 2076 2075 1793 1792 Nc|jdvs Jd||j|_|jdvs Jd||j|_||_||_||_|j %|sJdt|ttd||_|j||_ |jy)N)NGRAMSEQz,data type should be 'NGRAM', 'SEQ', but got )traintestz(mode should be 'train', 'test', but got z;data_file is not set and downloading automatically disabledimikolov) upper data_typelowermode window_size min_word_freq data_filerURLMD5_build_work_dictword_idx _load_anno)selfrrrrrdownloads fG:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/text/datasets/imikolov.py__init__zImikolov.__init__Ts %   F:) E F #*zz|   =6dV < = JJL &*" >> ! ML M73ZDN --m<  c|tjt}|D]L}|jj D]}||xxdz cc<|dxxdz cc<|dxxdz cc<N|S)N) collections defaultdictintstripsplit)rf word_freqlws r word_countzImikolov.word_count{so  #//4IAWWY__&! ! ' e  !  e  !   rc 6d}d}tj|j5}|j|}|j|}|j ||j |}d|vr|d=|j Dcgc]}|d|j kDs|}}t|d} tt| \} } ttt| tt| } t| | d<ddd| Scc}w#1swY SxYw)Nz$./simple-examples/data/ptb.train.txtz$./simple-examples/data/ptb.valid.txtr c|d |dfS)Nr r)xs rz+Imikolov._build_work_dict..s1qt}r)key) tarfileopenr extractfiler,itemsrsortedlistzipdictrangelen) rcutofftrain_filename test_filenametftrainftestfr)r1word_freq_sortedwords_rs rrzImikolov._build_work_dicts ?> \\$.. )R^^N3FNN=1Etv/FGI)#g&%??,,a!t7I7I0I,  &i5LM C!123HE1DUE#e*,=!>?@H #E HW *"*"s%ADD D A"D DDc ng|_tj|j5}d|jd}|j |}|j d}|D]}|jdk(r|jdkDsJddg|jjzdgz}t||jk\sh|Dcgc]}|j j|| }}t|jt|d zD]6}|jjt|||jz |8|jd k(r|jj}|Dcgc]}|j j|| }}|j dg|z}||j dgz} |jd kDrt||jkDr|jj|| ft!d  dddycc}wcc}w#1swYyxYw) Nz./simple-examples/data/ptb.z.txtr.r zInvalid gram lengthr!r"r r rzUnknow data type)datar4r5rrr6rrrr&r'r=getr<appendtupleAssertionError) rrAfilenamer(UNKr*r+isrc_seqtrg_seqs rrzImikolov._load_annos \\$.. )R4TYYKtDHx(A--(C>>W,++b0G2GG0!'')//"33ug=A1v!1!11@AB1T]]..q#6B!&t'7'7Q!!DA II,,U1Q9I9I5IA3N-OP"E^^u, )A<=>Aq**1c2AA>#}}U34q8G4==#7"88G''!+G t?O?O0O II$$gw%78();<<# * )C ?* )s2BH+H+#H!)BH+8#H&A:>abhhqk>:;;:s:c,t|jSrT)r=rI)rs r__len__zImikolov.__len__s499~r)Nr rHr 2TrT) __name__ __module__ __qualname____doc__rr,rrrYr[r0rrrrs;4p %N ,=4<rr) r#r4numpyrUpaddle.dataset.commonr paddle.ior__all__rrrr0rrres3< D(^w^r