o rq i~s@sddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlmZddlmZmZmZmZmZddlZddlmZddlmZdd lmZmZdd lmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*m+Z+dd l,m-Z-m.Z.dd l/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6m7Z7ddl8m9Z9ddl:m;Z;me(dddddfde?de?dee?dee?deee?dfdee@dee dee?dee?fddZAe'dddddfde?de?dee?dee?edfdee?deeee?fdee@dee de?fd d!ZBde(ddddddd"d#e?de?d$e?dee?dee?deee?dfdee@dee dee?d%e@dee?fd&d'ZCde?de?fd(d)ZDdde*fde?de?de?d$e?fd*d+ZE d>de?de?de?d,ee?fd-d.ZFd/d0ZG     d?d1e?de?d2e?de d3eee?e?fd4eHd%e@d5eee2d,e?fd6d7ZI   d@d1e?de?d2e?d4eHde d3eee?e?fd%e@d5eee2fd8d9ZJ d>d1e?de?d2e?de d3eee?e?ff d:d;ZK  dAd5eee2fdFile exists in local cache, but we're not sure it's up to datezCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z user-agent)r,)r;r2)r*r.endpointTF)r(r* recursiveZ use_cookiesr<r treerNamezFile z8 already in cache with identical hash, skip downloading!rd/)r;r* root_pathr= page_number page_sizer<z Get dataset: z file list failed, error: z"The file path: %s not exist in: %s) file_nameZ dataset_name namespacer*r<zInvalid repo type )rrr"$create_temporary_directory_and_cacheZget_file_by_pathloggerwarning ValueErrorr rZget_user_agentZ get_cookiesZget_endpoint_for_readZget_valid_revisionZget_model_filesexistsdebugZget_file_by_inforr'rZget_dataset_files Exceptionerrorlenr#get_file_download_urlZget_dataset_file_url download_file)r;r)r2r*r+r,r-r.r/r:temporary_cache_dircacheZcached_file_pathZ_apiheadersZ repo_filesr<Zfile_to_download_metaZ repo_filerEgroup_or_ownernamerCrDZ dataset_fileseZis_existZurl_to_downloadr4r4r5r3s               #  r3cCs|tjjr|tjj}tj|}tj|}|dkr dS|ds,tj|d}t|\}}| dd}tj|||}tj|||}tj |rstj |sut d|d|z t ||WdStyrYdSwdSdS)NZdatasetsZhub.___zLegacy cache dir exists: z , move to )endswithospathsepstripdirnamebasenamejoinr'replacerKrHinfoshutilmoverM)r+r(Zlegacy_cache_root base_namerUrVrRZlegacy_cache_dirr4r4r5!move_legacy_cache_to_standard_dir%s4       rgc Cs|tkrt}n|tkrt}ntd|t|\}}|dur-tj|t }t |}n)|dur8|}t ||t |t rAt|}tj|t ||}|dd}t |||}tj|dd||fS)Nz7repo_type only support model and dataset, but now is : rXrYTexist_ok)rrrrrJr'r[r\rarr$rg isinstancerstrrbmakedirs) r(r/r+r2Zdefault_cache_rootrUrVrRrSr4r4r5rGHs.       rGr<cCs8tj|}tj|}d}|st}|j||||dS)aFormat file download url according to `model_id`, `revision` and `file_path`. e.g., Given `model_id=john/bert`, `revision=master`, `file_path=README.md`, the resulted download url is: https://modelscope.cn/api/v1/models/john/bert/repo?Revision=master&FilePath=README.md Args: model_id (str): The model_id. file_path (str): File path revision (str): File revision. endpoint (str): The remote endpoint Returns: str: The file url. zQ{endpoint}/api/v1/models/{model_id}/repo?Revision={revision}&FilePath={file_path})r<r(r*r))urllibparse quote_plusr&format)r(r)r*r<Zdownload_url_templater4r4r5rPhs  rPc Cs|\}}}}}}}}|durint|} ttj| d<ttddgd} |d||f} zd} tj | r^t | d} | dt j} |D]}|| qGWdn1sYwY|| }||kriWdSd ||f| d <t | d 0} tj|d| |td }|jtd D]}|r| ||D] }|t|qqWdWdS1swYWdSty}z| jd||d} td||f| WYd}~nd}~wwq/)N X-Request-IDrGETtotalbackoff_factorallowed_methods_%s_%sTrrb bytes=%s-%sRangeab+streamrTr.timeout chunk_sizerNz-Downloading: %s failed, reason: %s will retry)copydeepcopyrkuuiduuid4hexr rr[r\rKopenseekioSEEK_ENDupdaterequestsgetr iter_contentrwriterOrM incrementrHrIsleep)paramsZmodel_file_pathprogress_callbacksstartendurlrEr.rT get_headersretrypart_file_namepartial_lengthfcallbackZdownload_startrchunkrWr4r4r5download_part_with_retrysn        rrrErT file_sizerc  s|durgn|}|s|tfdd|D}d} g} tj|} tjtj| ddtt | D]} | | } | d| d}| | || ||||fq9|dkrl| | ||dd|||ft t d}t |dd }t |t| Wdn1swY|D]}|qt}ttj|d N}| D]@}|d d |d |df}t|d} |dt}|sn ||||qWdn1swYt|qWd|S1swY|S)Ncg|]}|qSr4r4.0rrErr4r5  z%parallel_download..i Trhrdownload) max_workersthread_name_prefixwbrrwrx)rappendr r[r\rarlr_rangeintminrrlistmaprrhashlibsha256rreadrrrremove hexdigest)rr/rEr.rTrr:rr<Z PART_SIZEtasksr)idxrrZ parallelsexecutorr hash_sha256Z output_filetaskrZ part_filerr4rr5parallel_downloadsb             rc st|durgn|}|s|tfdd|D}|dur!int|}ttj|d<tj |} tj tj | ddt d|| d} t} ttd d gd } zd kr}t| d |D]} | d qdWdn1svwYWnd }tj | rd} t| d}|d tj}|D]} | |qWdn1swY|krWntd|d f|d<t| d9}tj|d||td}||jtdD]}|r|D] } | t|q| || s| |qWdn1swYWn"t!y%}zd} | j"d ||d} | #WYd}~nd}~wwqW|D]} | $q)| r6dS| %S)ajDownload remote file, will retry 5 times before giving up on errors. Args: url(str): actual download url of the file local_dir(str): local directory where the downloaded file stores file_name(str): name of the file stored in `local_dir` file_size(int): The file size. cookies(CookieJar): cookies used to authentication the user, which is used for downloading private repos headers(Dict[str, str], optional): http headers to carry necessary info when requesting the remote file disable_tqdm(bool, optional): Disable the progress bar with tqdm. progress_callbacks(List[Type[ProgressCallback]], optional): progress callbacks to track the download progress. Raises: FileDownloadError: File download failed. Ncrr4r4rrr4r5r rz'http_get_model_file..rqTrhdownloading %s to %sFrrrrsrzw+rxryrzr{r|rr)&rrr rrkrrrr[r\rarlr_rHrLrrr rrrrKrrrrrrraise_for_statusrrrOrrMrrrr)rr/rErr.rTr:rrZtemp_file_pathZ has_retryrrrrrrrrWr4rr5http_get_model_files!           / rc Csd}ttjd|dd}|durint|}ttj|d<|}t d||j t t dd gd } zS|} d | |d <tj|d ||td} | | jd} | dur[t| nd}tdd d|| d|dd} | jtdD]}|r| t|||qq| Wnty}z| jd ||d} | WYd}~nd}~wwq4Wdn1swYt d||tj !|j }||krt"|j d|||f}t #|t$|t%|j tj &||dS)ajDownload remote file, will retry 5 times before giving up on errors. Args: url(str): actual download url of the file local_dir(str): local directory where the downloaded file stores file_name(str): name of the file stored in `local_dir` cookies(CookieJar): cookies used to authentication the user, which is used for downloading private repos headers(Dict[str, str], optional): http headers to carry necessary info when requesting the remote file Raises: FileDownloadError: File download failed. rF)modedirdeleteNrqrrrrrsTz bytes=%d-rzr|zContent-LengthBiz Downloading [])unitZ unit_scaleZ unit_divisorrtinitialZdescrrzstoring %s in cache at %sz}File %s download incomplete, content_length: %s but the file downloaded length: %s, please download again)'rtempfileNamedTemporaryFilerrrkrrrrHrLrVr rtellrrrrrTrr rrrrOrcloserMrrr[r\getsizerrNr!rbra)rr/rEr.rTrtZtemp_file_managerr temp_filerZdownloaded_sizercontent_lengthprogressrrWZdownloaded_lengthmsgr4r4r5 http_get_filePs      (  rc Cstdd|dkr'tdkr't|||d||durdn||d||d}nt|||d|d||||d}tj||d} t|vr_|t} |durZ|| krYt dt | | nt | | | || S)NiSizerr)rTr.rr:r)rrTr.r:rzLMismatched real-time digest found, falling back to lump-sum hash computation) rrrget_dictrr[r\rarprintr%Zput_file) rZ file_metarRrSrTr.r:rZ file_digestrZ expected_hashr4r4r5rQsH      rQ)N)NNFNN)NFN)FN)Mrrrr[rdrrmrconcurrent.futuresr functoolsrhttp.cookiejarrpathlibrtypingrrrr r rZrequests.adaptersr Z tqdm.autor Zmodelscope.hub.apir rZmodelscope.hub.constantsrrrrrrrZmodelscope.utils.constantrrrrrZmodelscope.utils.file_utilsrrZmodelscope.utils.loggerrrrr errorsr!r"r#Z utils.cachingr$Z utils.utilsr%r&r'rHrkboolr6r9r3rgrGrPrrrrrrQr4r4r4r5sv      $    =  >     $ # 0  :  o Y