o p i@sddlZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z gZ ejdZeeejsH e ZejedddZddZeedd Zdd d Zd d ZdejfddZejfddZdddZdS)N~z.cachepaddledatasetc CsFzttWdSty"}z|jtjkrWYd}~dSd}~wwN)osmakedirs DATA_HOMEOSErrorerrnoEEXIST)pathexcr\/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/paddle/dataset/common.py must_mkdirs6s rcsDt}t|dtfdddD]}||q|S)Nrbcs dS)N)readrfrrDs zmd5file..)hashlibmd5openiterupdateclose hexdigest)fnameZhash_md5chunkrrrmd5fileAs   r!c Cstd|}t|trtd|}tjt|}tj |s't |tj||dur6| ddn|}tj |rGt ||krG|Sd}d}tj |rXt ||ksQtj |rmt jdt |d|d ||krv|d 7}n td |d |t jd |d|dt jdztjd|ddd}|jd} | durt|d} t|j| Wdn1swYn[t|dM} d} t| } | | d } | dkr| dnd } d}tjjj| dd}|j| dD]}| ||d 7}||i|| dkr||qWdn 1swYWdn 1s+wYWnt yD}zWYd}~qKd}~wwtj |rXt ||krXt jdt j!"|S)Nz^[a-zA-Z0-9_/\-]+$z^(?:(?!\.\.)[a-zA-Z0-9_/\.-])+$/rzfile z md5  zCannot download z within retry limit z Cache file z not found, downloading z zBegin to download GETT)timeoutZfollow_redirectszcontent-lengthwbritem)name) chunk_sizez Download finished )#rematchgroup isinstancestrrr joinrexistsrsplitr!sysstderrwrite RuntimeErrorhttpxstreamheadersgetrshutil copyfileobjrawintrZhapiZ progressbar ProgressBarZ iter_bytesr Exceptionstdoutflush)url module_nameZmd5sumZ save_namedirnamefilenameretryZ retry_limitr total_lengthrr-Z total_iterZ log_intervalZ log_indexbardataerrrdownloadJs               , rPcCsJddttjDD]}dttd|vr"td|q dS)NcSsg|] }|ds|qS)__) startswith).0xrrr s  zfetch_all..fetchzpaddle.dataset.)dirrr importlib import_modulerV)rGrrr fetch_alls rZz %05d.picklec Cst|stdg}d}t|D]5\}}||||krF||dkrFt||d}|||g}|d7}Wdn1sAwYq|rht||d}|||WddS1sawYdSdS)a you can call the function as: split(paddle.dataset.cifar.train10(), line_count=1000, suffix="imikolov-train-%05d.pickle") the output files as: |-imikolov-train-00000.pickle |-imikolov-train-00001.pickle |- ... |-imikolov-train-00480.pickle :param reader: is a reader creator :param line_count: line count for each file :param suffix: the suffix for the output files, should contain "%d" means the id for each file. Default is "%05d.pickle" :param dumper: is a callable function that dump object to file, this function will be called as dumper(obj, f) and obj is the object will be dumped, f is a file object. Default is cPickle.dump. zdumper should be callable.rwr&N)callable TypeError enumerateappendr) reader line_countsuffixZdumperlinesZindx_fidrrrrr5s$    "r5csfdd}|S)a Create a reader that yield element from the given files, select a file set according trainer count and trainer_id :param files_pattern: the files which generating by split(...) :param trainer_count: total trainer count :param trainer_id: the trainer rank id :param loader: is a callable function that load object from file, this function will be called as loader(f) and f is a file object. Default is cPickle.load c 3sts tdt}|g}t|D]\}}|kr.td|||q|D] }t|d}|}|EdHWdn1sLwYq1dS)Nzloader should be callable.z append file: rK)r\r]globsortr^printr_r) file_listZ my_file_listidxfnrrc files_patternloader trainer_count trainer_idrrr`s$     z$cluster_files_reader..readerr)rmrorprnr`rrlrcluster_files_readersrqTcCs8|r tj|r |S|rtjj|||St|d)Nz& not exists and auto download disabled)rr r4rrcommonrP ValueError)r rFrrGrPrrr_check_exists_and_downloads rtr)T)r rfrrXrpickler.r>r6tempfiler:rZpaddle.dataset__all__r expanduserHOMEaccessW_OK gettempdirr3rrr!rPrZdumpr5loadrqrtrrrrs8  D ' !