o W+ iJ@sxdZddlZddlmZddlmZddlmZmZm Z m Z m Z m Z m Z ddlmZmZddlmZddlmZmZmZdd lmZmZdd lmZdd lmZeZd d gZeddGdddZ eddGdddZ!eddGdddZ"eddGdddZ# d'de e e$efde#fddZ%dede&fddZ'd(ded e$d!e&fd"d#Z(d$ed e$de e"fd%d&Z)dS))zr%z%CachedFileInfo.blob_last_modified_strcCr)zi (property) Size of the blob file as a human-readable string. Example: "42.2K". rrr r"r"r#size_on_disk_strH zCachedFileInfo.size_on_disk_strN) __name__ __module__ __qualname____doc__str__annotations__rintfloatpropertyr$r&r(r"r"r"r#rs   rc@steZdZUdZeed<eed<eed<ee ed<e ed<e defdd Z e defd d Z e defd d ZdS)CachedRevisionInfoaFrozen data structure holding information about a revision. Args: commit_hash (`str`): Hash of the revision (unique). Example: `"9338f7b671827df886678df2bdd7cc7b4f36dffd"`. snapshot_path (`Path`): Path to the revision directory in the `snapshots` folder. It contains the exact tree structure as the repo on the Hub. files: (`FrozenSet[CachedFileInfo]`): Set of [`~CachedFileInfo`] describing all files contained in the snapshot. size_on_disk (`int`): Sum of the blob file sizes that are symlink-ed by the revision. last_modified (`float`): Timestamp of the last time the revision has been created/modified. commit_hash snapshot_pathrfiles last_modifiedrcCr)z (property) Timestamp of the last time the revision has been modified, returned as a human-readable string. Example: "2 weeks ago". rr7r r"r"r#last_modified_strlr%z$CachedRevisionInfo.last_modified_strcCrzn (property) Sum of the blob file sizes as a human-readable string. Example: "42.2K". r'r r"r"r#r(vr)z#CachedRevisionInfo.size_on_disk_strcCr)zC (property) Total number of files in the revision. )lenr6r r"r"r#nb_filess zCachedRevisionInfo.nb_filesN)r*r+r,r-r.r/rr0rrr1r2r9r(r<r"r"r"r#r3Rs   r3c@seZdZUdZeed<eed<eed<eed<eed<ee ed<e ed<e ed <e d efd d Z e d efd dZ e d efddZdS)CachedRepoInfoaFrozen data structure holding information about a cached repository. Args: repo_id (`str`): Repo id of the repo on the Hub. Example: `"damo/bert-base-chinese"`. repo_type (`Literal["dataset", "model"]`): Type of the cached repo. repo_path (`Path`): Local path to the cached repo. size_on_disk (`int`): Sum of the blob file sizes in the cached repo. nb_files (`int`): Total number of blob files in the cached repo. revisions (`FrozenSet[CachedRevisionInfo]`): Set of [`~CachedRevisionInfo`] describing all revisions cached in the repo. last_accessed (`float`): Timestamp of the last time a blob file of the repo has been accessed. last_modified (`float`): Timestamp of the last time a blob file of the repo has been modified/created. repo_id repo_type repo_pathrr< revisions last_accessedr7rcCr)z (property) Last time a blob file of the repo has been accessed, returned as a human-readable string. Example: "2 weeks ago". )rrBr r"r"r#last_accessed_strr%z CachedRepoInfo.last_accessed_strcCr)z (property) Last time a blob file of the repo has been modified, returned as a human-readable string. Example: "2 weeks ago". r8r r"r"r#r9r%z CachedRepoInfo.last_modified_strcCrr:r'r r"r"r#r(r)zCachedRepoInfo.size_on_disk_strN)r*r+r,r-r.r/rr0rr3r1r2rCr9r(r"r"r"r#r=s    r=c@sReZdZUdZeed<eeed<ee ed<e de fddZ de fdd Z d S) ModelScopeCacheInfoaFrozen data structure holding information about the entire cache-system. This data structure is returned by [`scan_cache_dir`] and is immutable. Args: size_on_disk (`int`): Sum of all valid repo sizes in the cache-system. repos (`FrozenSet[CachedRepoInfo]`): Set of [`~CachedRepoInfo`] describing all valid cached repos found on the cache-system while scanning. warnings (`List[CorruptedCacheException]`): List of [`~CorruptedCacheException`] that occurred while scanning the cache. Those exceptions are captured so that the scan can continue. Corrupted repos are skipped from the scan. rreposwarningsrcCr)zp (property) Sum of all valid repo sizes in the cache-system as a human-readable string. r'r r"r"r#r(s z$ModelScopeCacheInfo.size_on_disk_strcsNdtdtdttfddgd}fddt|jd d d D}t||d S) aGenerate a detailed table from the [`ModelScopeCacheInfo`] object. Returns a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "local_path". Example: ```py >>> from modelscope.hub.cache_manager import scan_cache_dir >>> ms_cache_info = scan_cache_dir() ModelScopeCacheInfo(...) >>> print(ms_cache_info.export_as_table()) REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED LOCAL PATH ---------------------- --------- ---------- ------------ -------- ------------- ------------------------------------------------------------- damo/bert-base-chinese model master 2.7M 5 1 week ago ~/.cache/modelscope/hub/models--damo--bert-base-chinese/... damo/structured-bert model master 8.8K 1 1 week ago ~/.cache/modelscope/hub/models--damo--structured-bert/... damo/t5-base model master 893.8M 4 7 months ago ~/.cache/modelscope/hub/models--damo--t5-base/... ``` Returns: `str`: The table as a string. reporevisionrc Ss.|j|j|jd|j|j|j|jt|j gS)zHFormat a single repo and revision into a list of strings for tabulation.z{:>12}) r>r?r4formatr(r<rCr9r.r@)rGrHr"r"r#format_repo_revisions zAModelScopeCacheInfo.export_as_table..format_repo_revision)zREPO IDz REPO TYPEZREVISIONz SIZE ON DISKzNB FILESZ LAST_ACCESSEDZ LAST_MODIFIEDz LOCAL PATHcs.g|]}t|jdddD]}||q qS)cS|jSN)r4)rHr"r"r#z@ModelScopeCacheInfo.export_as_table...key)sortedrA).0rGrHrJr"r# s z7ModelScopeCacheInfo.export_as_table..cSrKrL)r>)rGr"r"r#rMrNz5ModelScopeCacheInfo.export_as_table..rO)rowsheaders)r=r3rr.rQrEr)r!Zcolumn_headersZ table_datar"rSr#export_as_tables  z#ModelScopeCacheInfo.export_as_tableN)r*r+r,r-r0r/rr=rr r2r.r(rWr"r"r"r#rDs   rD cache_dirrc Cs|durt}t|}|std|d|d|r)td|dt}g}|d}|d}|rP| rPt |t d\}}| || ||rj| rjt |td\}}| || |t |t d d \} } | | | | tt|td d |D|d S)aScan the entire ModelScope cache-system and return a [`ModelScopeCacheInfo`] structure. Use `scan_cache_dir` to programmatically scan your cache-system. The cache will be scanned repo by repo. If a repo is corrupted, a [`~CorruptedCacheException`] will be thrown internally but captured and returned in the [`~ModelScopeCacheInfo`] structure. Only valid repos get a proper report. ```py >>> from modelscope.hub.utils import scan_cache_dir >>> ms_cache_info = scan_cache_dir() ModelScopeCacheInfo( size_on_disk=3398085269, repos=frozenset({ CachedRepoInfo( repo_id='damo/t5-small', repo_type='model', repo_path=PosixPath(...), size_on_disk=970726914, nb_files=11, revisions=frozenset({ CachedRevisionInfo( commit_hash='master', size_on_disk=970726339, snapshot_path=PosixPath(...), files=frozenset({ CachedFileInfo( file_name='config.json', size_on_disk=1197 file_path=PosixPath(...), blob_path=PosixPath(...), ), CachedFileInfo(...), ... }), ), CachedRevisionInfo(...), ... }), ), CachedRepoInfo(...), ... }), warnings=[ CorruptedCacheException("Snapshots dir doesn't exist in cached repo: ..."), CorruptedCacheException(...), ... ], ) ``` Args: cache_dir (`str` or `Path`, `optional`): Cache directory to scan. Defaults to the default ModelScope cache directory. Raises: `CacheNotFound`: If the cache directory does not exist. `ValueError`: If the cache directory is a file, instead of a directory. Returns: a [`ModelScopeCacheInfo`] object. NzCache directory not found: zQ. Please use `cache_dir` argument or set `MODELSCOPE_CACHE` environment variable.)rXz1Scan cache expects a directory but found a file: modelsdatasetsr?T)r?inplacecs|]}|jVqdSrL)r)rRrGr"r"r# z!scan_cache_dir..)rErrF)rr expanduserresolveexistsr is_file ValueErrorsetis_dir _scan_dirrupdateextendrrD frozensetsum) rXrErFZ model_dirZ dataset_dirZ model_reposZmodel_warningsZ dataset_reposZdataset_warningsZ other_reposZother_warningsr"r"r#scan_cache_dir#sL?        rldircCs6|sdS|s dS|rdS|jtvrdSdS)z+Check if a directory is valid for scanning.FT)rbrf is_symlinknameFILES_TO_IGNORE)rmr"r"r# _is_valid_dirs rqFr?r\c Cst}g}|D]C}|r|jdvrq t|sq |D]/}t|s#qzt||d}|dur3||WqtyK}z ||WYd}~qd}~wwq ||fS)zWScan a directory for cached repos and return a set of [`~CachedRepoInfo`] and warnings.)rYrZZhubr[N)reiterdirrorq_scan_cached_repoaddr append) rmr?r\rErFZ owner_dirZname_dirinfoer"r"r#rgs*     rgr@c s|s td|ztt|}|j}|j}|dd}|dkr'WdSWnty;}ztd|d}~wwit }|D]?}t j ||d} | dd } t j | s[qCt| } | | <|tt j |d| | | j| | j| jd qCd } |rd |vr|d d dd} |rtfdd|D} n|j} t| t|tfdd|D|| d}rtddD}tddD}n |}|j}|j}tt|||t|gtddD||dS)zScan a single cache repo and return information about it. Any unexpected behavior will raise a [`~CorruptedCacheException`]. zRepo path is not a directory: Z___.unknownNz"Failed to load cache information: rZRevision)rrrrrrrZmasterz Revision:,rc3|] }|jjVqdSrL)rst_mtimerRfileZ blob_statsr"r#r^z$_scan_cached_repo..c3r}rL)rst_sizerrr"r#r^r)r4r6rr5r7csr]rL)st_atimerRstatr"r"r#r^r_csr]rL)r~rr"r"r#r^r_csr]rL)rrr"r"r#r^r_)r<r>r@r?rArrBr7)rfr r r. cached_filescached_model_revisionZ get_model_idreplace Exceptionreospathjoingetrbrrrtrbasenamerrr~splitmaxr3rjrkvaluesr=r;)r@r?cacherrr>rwZcached_files_infoZ cached_filerrrZ revision_hashZrevision_last_modifiedZcached_revisionZrepo_last_accessedZrepo_last_modifiedZ repo_statsr"rr#rss         rsrL)F)*r-r dataclassesrpathlibrtypingrrrrrr r Zmodelscope.hub.errorsr r Zmodelscope.hub.utils.cachingr Zmodelscope.hub.utils.utilsrrrZmodelscope.utils.constantrrZmodelscope.utils.file_utilsrZmodelscope.utils.loggerrloggerrprr3r=rDr.rlboolrqrgrsr"r"r"r#sD  $   ;4>] s