\;j ddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z gZ ejjdZej"eej$s e j&Zejj)edddZdZeedZddZd Zd ej4fd Zej8fd Zdd Zy)N~z.cachepaddledatasetc tjty#t$r(}|jtj k7rYd}~yd}~wwxYwN)osmakedirs DATA_HOMEOSErrorerrnoEEXIST)pathexcs ^G:\00. PROJECTS\API\Inventory\templateJSON\kerjaOCR\Lib\site-packages\paddle/dataset/common.py must_mkdirsr6s; I  99 $  %s A AA ctj}t|dtfddD]}|j |j |j S)Nrbc&jdS)N)read)fsrzmd5file..Ds affTlr)hashlibmd5openiterupdateclose hexdigest)fnamehash_md5chunkrs @rmd5filer$AsP{{}H UDA*C01GGI    rctjd|j}t|tr$tjd|j}t j jt|}t j j|st j|t j j|||jddn|}t j j|rt||k(r|Sd}d}t j j|rt||k(st j j|r/tjjdt|d|d ||kr|d z }nt!d |d |tjjd |d|dtjjd t#j$d|dd5}|j&j)d} | 6t+|d5} t-j.|j0| dddnt+|d5} d} t3| } | | z d z} | dkDr| dznd } d}t4j6j8j;| d}|j=| D]D}| j||d z }|j?|i|| zdk(s4|j?|F ddddddt j j|st||k(stjjdtjBjE|S#1swYxYw#1swYxYw#1swYxYw#t@$r }Yd}~d}~wwxYw)Nz^[a-zA-Z0-9_/\-]+$z^(?:(?!\.\.)[a-zA-Z0-9_/\.-])+$/rzfile z md5  zCannot download z within retry limit z Cache file z not found, downloading z zBegin to download GETT)timeoutfollow_redirectszcontent-lengthwbritem)name) chunk_sizez Download finished )#rematchgroup isinstancestrrrjoinr existsr splitr$sysstderrwrite RuntimeErrorhttpxstreamheadersgetrshutil copyfileobjrawintrhapi progressbar ProgressBar iter_bytesr Exceptionstdoutflush)url module_namemd5sum save_namedirnamefilenameretry retry_limitr total_lengthrr2 total_iter log_interval log_indexbardataes rdownloadr^Js-((0+>DDFK)S!HH 0) %' ggll9k2G 77>>' " Gww||y'83#iH ww~~hGH$5$? EKww~~h'GH,=,G 77>>( # JJ  uWX%6$7vfXRH I ;  QJE"3%';K=I  (#;C5 D  ./ sD4 yy}}-=> 'h-**155!4.-h-%) '*<'8 %1J%>%B 0:RJ",Q%%& $kk55AA&VB%&LLJL$GDGGDM%NIJJy"5(<71< # 9 5 %H.ww~~h'GH,=,GXJJ,-JJ O5.-.-4   sa8N>*N2;!NN21BN&N&N2"N>N# N2&N/ +N22N;7N>> O OcttjDcgc]}|jdr|c}D]L}dtt j d|zvs't j d|zj Nycc}w)N__fetchzpaddle.dataset.%s)dirrr startswith importlib import_modulera)xrOs r fetch_allrgs~v~~&&aall4.@&  c  # #$7+$E F    # #$7+$E F L L N s B B z %05d.picklecpt|s tdg}d}t|D]M\}}|j|||k\s||zdk(s&t ||zd5}|||g}|dz }dddO|r"t ||zd5}|||dddyy#1swY~xYw#1swYyxYw)a you can call the function as: split(paddle.dataset.cifar.train10(), line_count=1000, suffix="imikolov-train-%05d.pickle") the output files as: |-imikolov-train-00000.pickle |-imikolov-train-00001.pickle |- ... |-imikolov-train-00480.pickle :param reader: is a reader creator :param line_count: line count for each file :param suffix: the suffix for the output files, should contain "%d" means the id for each file. Default is "%05d.pickle" :param dumper: is a callable function that dump object to file, this function will be called as dumper(obj, f) and obj is the object will be dumped, f is a file object. Default is cPickle.dump. zdumper should be callable.rwr*N)callable TypeError enumerateappendr) reader line_countsuffixdumperlinesindx_fidrs rr:r:s, F 455 E F&(#1 Q ?q:~2fvos+qua ! ,+$ &6/3 '1 5! ( ' ,+ ( 's B  B, B) ,B5cfd}|S)a Create a reader that yield element from the given files, select a file set according trainer count and trainer_id :param files_pattern: the files which generating by split(...) :param trainer_count: total trainer count :param trainer_id: the trainer rank id :param loader: is a callable function that load object from file, this function will be called as loader(f) and f is a file object. Default is cPickle.load c3vKts tdtj}|jg}t |D]-\}}|z k(st d|z|j |/|D])}t|d5}|}|Ed{ddd+y7#1swY8xYww)Nzloader should be callable.zappend file: %srV)rjrkglobsortrlprintrmr) file_list my_file_listidxfnrrr files_patternloader trainer_count trainer_ids rrnz$cluster_files_reader..readers89 9IIm,   +GC]"j0'",-##B',Bb#!q    !s6AB92B9 B-B+B- B9+B--B6 2B9)rrrrrns```` rcluster_files_readerrs ! Mrc|r!tjj|r|S|r+tjj j |||St|d)Nz& not exists and auto download disabled)rrr9rrcommonr^ ValueError)rrNrrOr^s r_check_exists_and_downloadrsN t$ ~~$$--c;DDD6!GHIIrr)T)r rxrrdrpickler3rCr;tempfiler?rpaddle.dataset__all__r expanduserHOMEaccessW_OK gettempdirr8r rr$r^rgdumpr:loadrrrrrrs     ww#ryyrww 8   D GGLLx9 =  I AHO&36;;#N6<[[BJr