o
    rqi                     @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dl m!Z! d dl"Z"d dl#Z#d d	l#m$Z$ d d
l%m&Z&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZK d dlLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZV d dlWmXZX d dlYmZZZm[Z[ d dl\m]Z] d dl^m_Z_ d dl`maZambZbmcZcmdZdmeZemfZfmgZg d dlhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZy d dlzm{Z{m|Z| d dl}m~Z~ d dlmZmZmZmZmZmZmZmZ d dlmZ e~ ZG dd dZG dd dZG dd dZdS )    N)defaultdict)
HTTPStatus)	CookieJar)
expanduserPath)AnyBinaryIODictIterableListOptionalTupleUnion)	urlencode)Session)HTTPAdapterRetry)	HTTPErrortqdm)API_HTTP_CLIENT_MAX_RETRIESAPI_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_MAX_WORKERSDEFAULT_MODELSCOPE_INTL_DOMAINMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_CREDENTIALS_PATHMODELSCOPE_DOMAINMODELSCOPE_PREFER_AI_SITEMODELSCOPE_REQUEST_IDMODELSCOPE_URL_SCHEMEONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODTEMPORARY_FOLDER_NAME"UPLOAD_BLOB_TQDM_DISABLE_THRESHOLDUPLOAD_COMMIT_BATCH_SIZEUPLOAD_MAX_FILE_COUNTUPLOAD_MAX_FILE_COUNT_IN_DIRUPLOAD_MAX_FILE_SIZE#UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT$UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFSDatasetVisibilityLicensesModelVisibility
VisibilityVisibilityMap)
InvalidParameterNotExistErrorNotLoginExceptionRequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper)DatasetInfo	ModelInfo
Repository)	AigcModel)add_content_to_file
get_domainget_endpointget_readable_folder_sizeget_release_datetimeis_env_truemodel_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORTConfigFieldsDatasetFormationsDatasetMetaFormatsDownloadChannelDownloadMode
Frameworks	ModelFileTasksVirgoDatasetConfig)get_file_hashget_file_size)
get_logger)DATASET_LFS_SUFFIXDEFAULT_IGNORE_PATTERNSMODEL_LFS_SUFFIXCommitHistoryResponse
CommitInfoCommitOperationCommitOperationAdd	RepoUtils)thread_executorc                   @   s  e Zd ZdZdeefdee fddZdefddZ	dd
ee
 fddZ		ddee dee fddZejejdddddfdedee dee dee dee dee dee ded defddZ			ddededee dee ded defddZddedee fddZddedee fd d!Zedfded"ee dee defd#d$Zdd%d&ed'ee defd(d)Zedd*d&ed"ee dee defd+d,Zddd*d&ed"ee dee defd-d.Zeedd/d&ed'ee d"ee dee deeef f
d0d1Zddd	dd2d&ed'ee dee d3ee
 dee de
fd4d5Z dd&ed'edee fd6d7Z!e"d8d9 Z#ejejdd:de$dddf	ded;edee dee dee d<ee d=ee d"ee dee d>eee%e ef  d?eeee%e f  fd@dAZ&	B	C	ddDedEee dFee dee de'f
dGdHZ(ddIee
e)f de)fdJdKZ*			ddedLee dIee
e)f de%e fdMdNZ+				ddedLee dIee
e)f dee de%e f
dOdPZ,dQdR Z-			ddedSee) dee fdTdUZ.			ddedSee) dee fdVdWZ/			ddedIee
e)f dee de0e%e e%e f fdXdYZ1		ddedIee
e)f de0e%e e%e f fdZd[Z2edd	d	i dfded"ee d\ee d]ee
 dIee
e)f d^ee' dee de%e' fd_d`Z3ddad&edbed"ee fdcddZ4deje5jddfdeedfedee dee dee dgee dee defdhdiZ6ddee fdjdkZ7ddledee fdmdnZ8ddeedfedee fdodpZ9	q	B	r	ddeedfed"edsed]e
dEedFedee fdtduZ:ee$dBdvddwd&ed'ee d"ee dEedFedee fdxdyZ;e$dzdqdBdrdd{d&ed"edsed]e
dEedFedee fd|d}Z<e$dfdled"ee dee fd~dZ=	ddeedfedled"edee f
ddZ>e"dedefddZ?	ddeedfed"ededede@dee fddZAe"deBjCfddZDeEd	dqdfdedeedfed"ee dee
 dee
 dee fddZFeEdfdedeedfed"ee dee f
ddZGeEdfdeedfed"ee dee fddZHeEdfdeedfede
d"ee dee f
ddZIddledede'fddZJ	ddeedfed"ededee f
ddZK	ddee fddZL	ddedeedfed"edee defddZM	ddedeedfed"edee defddZNdd ZO	ddeedfede
dee ddf
ddZPdd ZQdd&edee defddZRdeSjedejdd	dqdd	d&edeee
df dee d'ee dee dee dee dee
 dee
 deeT defddZUddee$ddddd&edeVeW d<edee ded'ee d"ee dee dededeXfddZYdedddBdd	e$ddeeeZe[e\f ded&edeedf d'ee d<ee dee dee dee dee
 d"ee deXfddĄZ]ddddedde^e$dŜ	d&edeeeZe%e e%eZ f dee d<ee dee deedf d'ee deee%e ef  deee%e ef  ded"ee deeXe%eX f fdd˄Z_d	ddBd̜d&ed'edededeeeZe[e\f dee
 dee dee de'fddфZ`ddҜd&ed'ede%eaeebf  dee de%eaeebf  f
ddՄZc		ddeeeZe%e e%eZ f dedeee%e ef  deee%e ef  de%eede@f  f
dd؄Zee"deVeW d<edeaeebf fddڄZfddegfddބZhedd*d&ed'edeee%e f d"ee dee deaeebf fddZidS )HubApizModel hub api interface.
    Nendpointc                 C   s   |dur|nt  | _dt i| _t | _t|dddddd}t|d}| j	d	| | j	d
| t
D ]}t| j|tjt| j||d q4t | _dS )u   The ModelScope HubApi。

        Args:
            endpoint (str, optional): The modelscope server http|https address. Defaults to None.
        N
user-agent      )  i  i  i  F)totalreadconnectbackoff_factorstatus_forcelistrespect_retry_after_header)max_retrieszhttp://zhttps://timeout)rH   rk   ModelScopeConfigget_user_agentheadersr   sessionr   r   mountr(   setattr	functoolspartialgetattrUploadingCheckupload_checker)selfrk   rx   rv   retryadaptermethod r   Y/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/modelscope/hub/api.py__init__e   s.   	

zHubApi.__init__access_tokenc                 C   sL   ddl m} ddlm} | jr|| jjnt }| }|jd||dd |S )z
        Get jar cookies for authentication from access_token.

        Args:
            access_token (str): user access token on ModelScope.

        Returns:
            jar (CookieJar): cookies for authentication.
        r   )RequestsCookieJar)urlparsem_session_id/)domainpath)Zrequests.cookiesr   urllib.parser   rk   netlocrG   set)r   r   r   r   r   jarr   r   r   _get_cookies   s   
zHubApi._get_cookiesFcookies_requiredc                 C   s2   |r	| j |d}nt }|du r|rtd|S )a  
        Get cookies for authentication from local cache or access_token.

        Args:
            access_token (str): user access token on ModelScope
            cookies_required (bool): whether to raise error if no cookies found, defaults to `False`.

        Returns:
            cookies (CookieJar): cookies for authentication.

        Raises:
            ValueError: If no credentials found and cookies_required is True.
        )r   NzNo credentials found.You can pass the `--token` argument, or use HubApi().login(access_token=`your_sdk_token`). Your token is available at https://modelscope.cn/my/myaccesstoken)r   ry   get_cookies
ValueError)r   r   r   cookiesr   r   r   r      s   zHubApi.get_cookiesc                 C   s   |du r
t jd}|sdS |s| j}| d}| jj|d|i| | jd}t| |	 }t
| |t t }|j}t| t| t|t t |t t  |t t |fS )a|  Login with your SDK access token, which can be obtained from
           https://www.modelscope.cn user center.

        Args:
            access_token (str): user access token on modelscope, set this argument or set `MODELSCOPE_API_TOKEN`.
            If neither of the tokens exist, login will directly return.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            cookies: to authenticate yourself to ModelScope open-api
            git_token: token to access your git repository.

        Note:
            You only have to login once within 30 days.
        NZMODELSCOPE_API_TOKENNNz/api/v1/loginZAccessToken)jsonr{   )osenvirongetrk   r|   postbuilder_headersr{   r>   r   r?   r   r   r   ry   
save_tokensave_cookiessave_user_infor   r   )r   r   rk   r   rdtokenr   r   r   r   login   s8   





zHubApi.login model_id
visibilitylicensechinese_nameoriginal_model_idr   
aigc_modelrE   returnc	                 C   s`  |du rt d| j|dd}	|s| j}t|\}
}|
|||||tjddd}|dure| d}|j|	| | j	|d	 |
|j|j|j|j|j|jpN||j|j|j|j|jd
 |jrd|j|d< n| d}| | j	}tdd }|dd| rd|d< | jj|||	|d}t| | }t| | d| }|dur| | || |S )a  Create model repo at ModelScope Hub.

        Args:
            model_id (str): The model id in format {owner}/{name}
            visibility (int, optional): visibility of the model(1-private, 5-public), default 5.
            license (str, optional): license of the model, default apache-2.0.
            chinese_name (str, optional): chinese name of the model.
            original_model_id (str, optional): the base model id which this model is trained from
            endpoint: the endpoint to use, default to None to use endpoint specified in the class
            token (str, optional): access token for authentication
            aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation.
                If provided, will create an AIGC model with automatic file upload.
                Refer to modelscope.hub.utils.aigc.AigcModel for details.

        Returns:
            str: URL of the created model repository

        Raises:
            InvalidParameter: If model_id is invalid or required AIGC parameters are missing.
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nmodel_id is required!Tr   r   ZMODELSCOPE_TRAIN_IDr   )r   NameChineseNamer4   LicenseZOriginalModelIdZTrainIdz/api/v1/models/aigcr   r{   rk   )TagShowNameCoverImagesZAigcTypeZTagDescriptionZVisionFoundationZ	BaseModelWeightsNameWeightsSha256WeightsSizeZ	ModelPathTriggerWordsZOfficialTagsz/api/v1/models.r   en_USzX-Modelscope-Accept-Languager   r   r{   /models/)!r6   r   rk   rL   r   r   r   preupload_weightsr   r{   updatetagcover_imagesZ	aigc_typedescriptionZbase_model_typeZbase_model_idweight_filenameweight_sha256weight_sizeZ
model_pathtrigger_wordsZofficial_tagsr   splitrstripendswithr|   r   r>   r   r?   Zupload_to_repo)r   r   r   r   r   r   rk   r   r   r   owner_or_groupnamebodyr   r{   Zintl_endr   r   Zmodel_repo_urlr   r   r   create_model   sf   !


zHubApi.create_modeltag_namec              	   C   s  |du rt d|du rt d| dv rt d| d| j|dd}|s*| j}t|\}}|durV| d	}	|j|| | j|d
 |j||||j	|j
|j|jd}
n| d| d}	d}||d}
| jj|	|
|| | jd}t| | }t| | d| d| }|S )a]  Create a tag for a model at ModelScope Hub.

        Args:
            model_id (str): The model id in format {owner}/{name}
            tag_name (str): The tag name (e.g., "v1.0.0")
            endpoint: the endpoint to use, default to None to use endpoint specified in the class
            token (str, optional): access token for authentication
            aigc_model (AigcModel, optional): AigcModel instance for AIGC model tag creation.
                If provided, will create an AIGC model tag with automatic parameters.
                Refer to modelscope.hub.utils.aigc.AigcModel for details.

        Returns:
            str: URL of the created tag

        Raises:
            InvalidParameter: If model_id, tag_name, ref, or description is invalid.
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nr   ztag_name is required!)mainmasterz
tag_name "zp" is not allowed. Please use a different tag name (e.g., "v1.0", "v1.1", "latest"). Reserved names: main, masterTr   z/api/v1/models/aigc/repo/tagr   )r   r   r   r   r   r   r   r   /api/v1/models/z	/repo/tagr   )ZTagNameRefr   r   z/tags/)r6   lowerr   rk   rL   r   r   r{   r   r   r   r   r   r|   r   r>   r   r?   )r   r   r   rk   r   r   r   r   r   r   r   revisionr   r   Ztag_urlr   r   r   create_model_tagP  sP   


zHubApi.create_model_tagc                 C   sb   t  }|s	| j}|du rtd| d| }| jj||| | jd}t| t	|
  dS )a0  Delete model_id from ModelScope.

        Args:
            model_id (str): The model id.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Raises:
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        N)Token does not exist, please login first.r   r   r{   ry   r   rk   r   r|   deleter   r{   r>   r?   r   )r   r   rk   r   r   r   r   r   r   delete_model  s   
zHubApi.delete_modelc                 C   s   |s| j }| d| dS )Nr   z.gitrk   )r   r   rk   r   r   r   get_model_url  s   zHubApi.get_model_urlr   c           	      C   s   t  }t|\}}|s| j}|r| d| d| d| }n
| d| d| }| jj||| | jd}t|t	|| |j
tjkrWt| rO| t S t| t t| dS )a  Get model information at ModelScope

        Args:
            model_id (str): The model id.
            revision (str optional): revision of model.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            The model detail information.

        Raises:
            NotExistError: If the model is not exist, will throw NotExistError

        Note:
            model_id = {owner}/{name}
        r   r   
?Revision=r   N)ry   r   rL   rk   r|   r   r   r{   r<   loggerstatus_coder   OKr=   r   r   r7   r   r>   )	r   r   r   rk   r   r   r   r   r   r   r   r   	get_model  s    

zHubApi.get_model)	repo_typerepo_idr   c                C   s  t jt}|dur5| dkr5t| }z| j|||dd W |S  ty4   t	d| d| d  w t
t }t|d}| j|||d	st| d}td| d
| d| d z| j|||dd W |S  ty~   t	d| d| d|   w |S )a  Get proper endpoint for read operation (such as download, list etc.)
        1. If user has set MODELSCOPE_DOMAIN, construct endpoint with user-specified domain.
           If the repo does not exist on that endpoint, throw 404 error, otherwise return the endpoint.
        2. If domain is not set, check existence of repo in cn-site and ai-site (intl version) respectively.
           Checking order is determined by MODELSCOPE_PREFER_AI_SITE.
             a. if MODELSCOPE_PREFER_AI_SITE is not set ,check cn-site first before ai-site (intl version)
             b. otherwise check ai-site before cn-site
           return the endpoint with which the given repo_id exists.
           if neither exists, throw 404 error
        Nr   T)r   r   rk   re_raiseRepo z does not exist on r   )Zcn_site)r   rk   z not exists on z#, will try on alternative endpoint )r   rk   r   z not exists on either z or )r   r   r   r#   stripr&   repo_exists	Exceptionr   errorrK   r$   rH   warning)r   r   r   srk   Zcheck_cn_firstZprefer_endpointZalternative_endpointr   r   r   get_endpoint_for_read  s:   


zHubApi.get_endpoint_for_read)r   rk   c          
      C   s^   t |\}}| j|||d}| j|t||d}| j||d|d}tdi ||||d}	|	S )aD  Get model information including commit history.

        Args:
            repo_id (str): The model id in the format of
                ``namespace/model_name``.
            revision (str, optional): Specific revision of the model.
                Defaults to ``DEFAULT_MODEL_REVISION``.
            endpoint (str, optional): Hub endpoint to use. When ``None``,
                use the endpoint specified when initializing :class:`HubApi`.

        Returns:
            ModelInfo: The model detailed information returned by
            ModelScope Hub with commit history.
        )r   r   rk   r   r   r   rk   T)r   r   	recursiverk   commitsZauthorsiblingsNr   )rL   r   list_repo_commitsrS   get_model_filesrB   )
r   r   r   rk   r   _Z
model_datar   r   
model_infor   r   r   r     s   zHubApi.model_infoc          
      C   sb   t |\}}| j|||d}| j|t||d}| j||ptd|d}tdi ||||d}	|	S )a>  Get dataset information including commit history.

        Args:
            repo_id (str): The dataset id in the format of
                ``namespace/dataset_name``.
            revision (str, optional): Specific revision of the dataset.
                Defaults to ``None``.
            endpoint (str, optional): Hub endpoint to use. When ``None``,
                use the endpoint specified when initializing :class:`HubApi`.

        Returns:
            DatasetInfo: The dataset detailed information returned by
            ModelScope Hub with commit history.
        )
dataset_idr   rk   r   T)r   r   r   rk   r   Nr   )rL   get_datasetr   rR   get_dataset_filesrM   rA   )
r   r   r   rk   r   r   Zdataset_datar   r   dataset_infor   r   r   r   9  s   zHubApi.dataset_info)r   r   rk   c                C   sN   |du s|t kr| j|||dS |tkr| j|||dS td| dt d)aR  Get repository information for models or datasets.

        Args:
            repo_id (str): The repository id in the format of
                ``namespace/repo_name``.
            revision (str, optional): Specific revision of the repository.
                Currently only effective for model repositories. Defaults to
                ``DEFAULT_MODEL_REVISION``.
            repo_type (str, optional): Type of the repository. Supported
                values are ``"model"`` and ``"dataset"``. If not provided,
                ``"model"`` is assumed.
            endpoint (str, optional): Hub endpoint to use. When ``None``,
                use the endpoint specified when initializing :class:`HubApi`.

        Returns:
            Union[ModelInfo, DatasetInfo]: The repository detailed information
            returned by ModelScope Hub.
        N)r   r   rk   Arg repo_type z# not supported. Please choose from r   )rS   r   rR   r   r6   rT   )r   r   r   r   rk   r   r   r   	repo_infoY  s   zHubApi.repo_info)r   rk   r   r   r   c                C   s  |du r| j }|dur| tvrtd| |du s"|ddkr(td| | j|dd}t|\}}|durJ| tkrJ| d| d| }	n
| d	| d| }	| jj	|	|| 
| jd
}
t|
t||d}|dkrodS |dkr{|ryt|
dS td| d td| )a  
        Checks if a repository exists on ModelScope

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`, *optional*):
                `None` or `"model"` if getting repository info from a model. Default is `None`.
                TODO: support studio
            endpoint(`str`):
                None or specific endpoint to use, when None, use the default endpoint
                set in HubApi class (self.endpoint)
            re_raise(`bool`):
                raise exception when error
            token (`str`, *optional*): access token to use for checking existence.
        Returns:
            True if the repository exists, False otherwise.
        NzNot support repo-type: %sr   rn   z5Invalid repo_id: %s, must be of format namespace/nameFr   /api/v1/datasets/r   r      Ti  z%Check repo_exists return status code r   zOFailed to check existence of repo: %s, make sure you have access authorization.)rk   r   rT   r   countr   rL   rR   r|   r   r   r{   r<   r   r   warn)r   r   r   rk   r   r   r   r   r   r   r   coder   r   r   r   |  s6   

zHubApi.repo_existsc                 C   s\   |s| j }|tkr| || n|tkr| || ntd| dtd| d dS )a  
        Delete a repository from ModelScope.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`):
                The type of the repository. Supported types are `model` and `dataset`.
            endpoint(`str`):
                The endpoint to use. If not provided, the default endpoint is `https://www.modelscope.cn`
                Could be set to `https://ai.modelscope.ai` for international version.
        r   z not supported.r   z deleted successfully.N)rk   rR   delete_datasetrS   r   r   r   info)r   r   r   rk   r   r   r   delete_repo  s   zHubApi.delete_repoc                 C   s`   t j| tj}tjtjtj	t
ji}t|d}t|| W d    d S 1 s)w   Y  d S )Nw)r   r   joinr[   CONFIGURATIONrU   	frameworkrZ   Ztorchtaskr\   otheropenr   dump)	model_dircfg_filecfgfiler   r   r   _create_default_config  s   "zHubApi._create_default_configzupload modelr  commit_messager   ignore_file_pattern
lfs_suffixc              
      s8  t jdtdd 	 |d u rtd|d u rtdtj|r%tj|r)tdtj|t	j
}tj|sItdt	j
 d| d	 t| t }|d u rUtd
t|}t|}|
d u rdg }
t|
trl|
g}
|d u st|d u rxtd| |std|  | j|||||	d tj|t}t }td| d| d td| d zzt||d}||}||vrtd|  ||| ||| t|}|D ]"  d dkrtj| }tj|rt| qt j!|dd q|D ]6  d dkr3t" fdd|
D rqtj| }tj#|r-t $|tj|  qt %|| q|sEt&j&' (d}d||f }|d ur`t|trS|gn|}|D ]}|)| qW|j*|||d |d urs|+|| td| d| d W n	 t,y    w W t j!|dd d S t j!|dd w ) NzThis function is deprecated and will be removed in future versions. Please use git command directly or use HubApi().upload_folder insteadrm   )
stacklevelzmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.zNo z file found in z, creating a default one.zMust login before upload!z5Visibility and License cannot be empty for new model.zCreating new model [%s])r   r   r   r   r   zPushing folder z
 as model r   zTotal folder size z;, this may take a while depending on actual pushing size...)r  Z
clone_fromzCreating new branch %sr   T)ignore_errorsc                    s   g | ]
}t | d uqS N)research).0patternfr   r   
<listcomp>P      z%HubApi.push_model.<locals>.<listcomp>z%Y_%m_%d_%H_%M_%Sz$[automsg] push model %s to hub at %s)r  Zlocal_branchZremote_branchzSuccessfully push folder z to remote repo [z].)-warningsr  DeprecationWarningr6   r   r   existsisfiler	  r[   r
  r   r   rj   r  ry   r   r8   listdirrI   
isinstancestrr   r  r   r)   r@   rD   Zget_remote_branchesZ
new_branchcheckoutremoveshutilrmtreeanyisdircopytreecopydatetimenowstrftimeZadd_lfs_typepushZtag_and_pushr   )r   r   r  r   r   r   r  r   r   r   r  r  r  r   Zfiles_to_saveZfolder_sizeZtmp_dirZgit_wrapperrepobranchesZfiles_in_reposrcdateZlfs_suffix_listsuffixr   r  r   
push_model  s   /







"zHubApi.push_modelrn   
   r   page_number	page_sizec           	      C   s   t  }|s	| j}| d}| jj|d|||f || | jd}t|t|| |j	t
jkrDt| r<| t }|S t| t t| dS )a  List models in owner or group.

        Args:
            owner_or_group(str): owner or group.
            page_number(int, optional): The page number, default: 1
            page_size(int, optional): The page size, default: 10
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Raises:
            RequestError: The request error.

        Returns:
            dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group}
        r   z.{"Path":"%s", "PageNumber":%s, "PageSize": %s})datar   r{   N)ry   r   rk   r|   putr   r{   r<   r   r   r   r   r=   r   r   r9   r   r>   )	r   r   r=  r>  rk   r   r   r   r?  r   r   r   list_modelsk  s(   

zHubApi.list_modelsuse_cookiesc                 C   s@   d }t |tr|}|S t |trt }|r|d u rtd|S )Nr   )r(  r   boolry   r   r   )r   rB  r   r   r   r   _check_cookie  s   

zHubApi._check_cookiecutoff_timestampc                 C   s.   | j |||d}|rdd |D }|S g }|S )a3  Get model branch and tags.

        Args:
            model_id (str): The model id
            cutoff_timestamp (int): Tags created before the cutoff will be included.
                                    The timestamp is represented by the seconds elapsed from the epoch time.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        )r   rE  rB  c                 S      g | ]}|d  qS Revisionr   r  xr   r   r   r!        z/HubApi.list_model_revisions.<locals>.<listcomp>)list_model_revisions_detail)r   r   rE  rB  Ztags_detailstagsr   r   r   list_model_revisions  s   zHubApi.list_model_revisionsc           
      C   s   |  |}|du rt }|s| j}| d| d| }| jj||| | jd}t|t|| |	 }t
| |t }	|	d d S )a  Get model branch and tags.

        Args:
            model_id (str): The model id
            cutoff_timestamp (int): Tags created before the cutoff will be included.
                                    The timestamp is represented by the seconds elapsed from the epoch time.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        Nr   z/revisions?EndTime=%sr   RevisionMapTags)rD  rJ   rk   r|   r   r   r{   r<   r   r   r?   r   )
r   r   rE  rB  rk   r   r   r   r   r  r   r   r   rL    s   


z"HubApi.list_model_revisions_detailc                 C   s"   |D ]}|d |kr|  S qd S )NrH  r   )r   detailsr   itemr   r   r   get_branch_tag_detail  s
   zHubApi.get_branch_tag_detailr   c                    s6  |s| j }t  tttj  }| j||d u rdn||d\}}|r,dd |D ng }|r7dd |D ng }	 |t kru|d u rLt	}t
d|  ||vr\||	vr\td||f | ||}
|
d u rl| ||}
t
d|  |
S |d ur||v r| ||}
|
S t|d	kr|d u s|t	krt	}ntd
||f | ||}
|
S |d u r|r fdd|D ng }t|d	kr|d	 d }|d	 }
nt	}| ||}
dd|	 }t
d|  t
d|  |
S ||	vr|t	krt
d | ||}
ndd|	 }td|||f | ||}
t
d|  |
S )NF)rB  rk   c                 S   rF  rG  r   rI  r   r   r   r!    rK  z4HubApi.get_valid_revision_detail.<locals>.<listcomp>c                 S   rF  rG  r   rI  r   r   r   r!    rK  z9Model revision not specified, using default [%s] version.z$The model: %s has no revision : %s .z!Development mode use revision: %sr   z#The model: %s has no revision: %s !c                    s   g | ]
}|d   kr|qS )Z	CreatedAtr   rI  Zrelease_timestampr   r   r!    s    rH  z[%s],z5Model revision should be specified from revisions: %sz.Model revision not specified, use revision: %sz?Using the master branch is fragile, please use it with caution!z0The model: %s has no revision: %s valid are: %s!z%Use user-specified model revision: %s)rk   rJ   introundr2  r3  	timestamp#get_model_branches_and_tags_detailsr'   rP   r   r  r7   rS  debuglenr	  r   )r   r   r   r   rk   Zcurrent_timestampZall_branches_detailZall_tags_detailZall_branchesZall_tagsZrevision_detailZrevisions_detailZvlr   rT  r   get_valid_revision_detail  sx   
'


z HubApi.get_valid_revision_detailc                 C   s   | j ||||dd S )N)r   r   r   rk   rH  )r\  )r   r   r   r   rk   r   r   r   get_valid_revision"  s   zHubApi.get_valid_revisionc           	      C   s|   |  |}|s
| j}| d| d}| jj||| | jd}t|t|| | }t	| |t
 }|d d |d d fS )a  Get model branch and tags.

        Args:
            model_id (str): The model id
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        r   z
/revisionsr   rO  ZBranchesrP  )rD  rk   r|   r   r   r{   r<   r   r   r?   r   )	r   r   rB  rk   r   r   r   r   r  r   r   r   rY  ,  s   


z*HubApi.get_model_branches_and_tags_detailsc                 C   sF   | j ||d\}}|rdd |D ng }|rdd |D ng }||fS )aq  Get model branch and tags.

        Args:
            model_id (str): The model id
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        )r   rB  c                 S   rF  rG  r   rI  r   r   r   r!  Z  rK  z6HubApi.get_model_branches_and_tags.<locals>.<listcomp>c                 S   rF  rG  r   rI  r   r   r   r!  \  rK  )rY  )r   r   rB  Zbranches_detailZtags_detailr7  rM  r   r   r   get_model_branches_and_tagsI  s   
z"HubApi.get_model_branches_and_tagsrootr   r{   c                 C   s  |s| j }|rd||||f }nd|||f }| |}	|dur'|d|  }|du r.| jn|}tt j|d< | jj||	|d}
t	|
t
|	| |
 }t| g }|t d sgt
d| d	| d
 |S |t d D ]}|d dks{|d dkr|qm|| qm|S )a  List the models files.

        Args:
            model_id (str): The model id
            revision (Optional[str], optional): The branch or tag name.
            root (Optional[str], optional): The root path. Defaults to None.
            recursive (Optional[bool], optional): Is recursive list files. Defaults to False.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            headers: request headers
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            List[dict]: Model file list.
        z7%s/api/v1/models/%s/repo/files?Revision=%s&Recursive=%sz+%s/api/v1/models/%s/repo/files?Recursive=%sNz&Root=zX-Request-IDr   FileszNo files found in model z at revision r   r   z
.gitignorez.gitattributes)rk   rD  r{   r)  uuiduuid4hexr|   r   r<   r   r   r?   r   r   append)r   r   r   r_  r   rB  r{   rk   r   r   r   r   filesr  r   r   r   r   `  s:   
zHubApi.get_model_files)r   filenamec                C   s<   t  }| j|d||du rdn|d}dd |D }||v S )ap  Get if the specified file exists

        Args:
            repo_id (`str`): The repo id to use
            filename (`str`): The queried filename, if the file exists in a sub folder,
                please pass <sub-folder-name>/<file-name>
            revision (`Optional[str]`): The repo revision
        Returns:
            The query result in bool value
        TNF)r   r   rB  c                 S   rF  r   r   )r  r  r   r   r   r!    rK  z&HubApi.file_exists.<locals>.<listcomp>)ry   r   r   )r   r   rf  r   r   re  r   r   r   file_exists  s   zHubApi.file_existsdataset_name	namespacer   c                 C   s   |d u s|d u rt dt }|d u rtd|s| j}| d}	d |fd |fd |fd |fd |fd |fd}
| jj|	|
|| | jd}t	||	|
 t
|  | d| d| }td|  |S )	Nz(dataset_name and namespace are required!r   /api/v1/datasets)r   r   ZOwnerr   r4   Description)re  r   r{   z
/datasets/r   zCreate dataset success: )r6   ry   r   r   rk   r|   r   r   r{   r;   r?   r   r   r  )r   rh  ri  r   r   r   r   rk   r   r   re  r   Zdataset_repo_urlr   r   r   create_dataset  s4   	
	
zHubApi.create_datasetc                 C   sT   |s| j }| d}i }| jj||| | jd}t| | t }dd |D S )Nrj  )paramsr{   c                 S   rF  )r   r   rI  r   r   r   r!    rK  z(HubApi.list_datasets.<locals>.<listcomp>)rk   r|   r   r   r{   r>   r   r   )r   rk   r   rm  r   Zdataset_listr   r   r   list_datasets  s   


zHubApi.list_datasetsr   c                 C   sb   t  }|s	| j}|d u rtd| d| }| jj||| | jd}t| t	|
  d S )Nr   r   r   r   )r   r   rk   r   r   r   r   r   r   r    s   
zHubApi.delete_datasetc           
      C   sj   |s| j }| d| d| }t }| jj||d}| }t||| |d d }|d d }	||	fS )z Get the dataset id and type. r   r   r   DataZIdType)rk   ry   r   r|   r   r   r:   )
r   rh  ri  rk   datahub_urlr   r   respr   dataset_typer   r   r   get_dataset_id_and_type  s   zHubApi.get_dataset_id_and_typeTd   	root_pathc	                 C   s   t dt | j|||d\}	}
|rdnd}|s| j}| d|	 d}|r'|nd|r,|nd|||d	}t }| jj|||d
}|	 }t
||| |S )z?
        @deprecated: Use `get_dataset_files` instead.
        zMThe function `list_repo_tree` is deprecated, use `get_dataset_files` instead.rh  ri  rk   TrueFalser   
/repo/treer   r   rH  ZRootZ	Recursive
PageNumberPageSizerm  r   )r#  r  r$  ru  rk   ry   r   r|   r   r   r:   )r   rh  ri  r   rw  r   r=  r>  rk   dataset_hub_idrt  rr  rm  r   r   rs  r   r   r   list_repo_tree   s&   

zHubApi.list_repo_tree2   )r   r   r=  r>  rk   c             
   C   s  ddl m} ||r|ddkr|d\}}	ntd| d|s&| j}|r3| d| d| d	n| d
| d	}
|p>t||d}t }z+| j	j
|
||| | jd}t| | }t| |
dtjkrpt|W S W dS  tjjy } ztd| dt| d}~ww )a@  
        Get the commit history for a repository.

        Args:
            repo_id (str): The repository id, in the format of `namespace/repo_name`.
            repo_type (Optional[str]): The type of the repository. Supported types are `model` and `dataset`.
            revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
            page_number (int): The page number for pagination. Defaults to 1.
            page_size (int): The number of commits per page. Defaults to 50.
            endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

        Returns:
            CommitHistoryResponse: The commit history response.

        Examples:
            >>> from modelscope.hub.api import HubApi
            >>> api = HubApi()
            >>> commit_history = api.list_repo_commits('meituan/Meeseeks')
            >>> print(f"Total commits: {commit_history.total_count}")
            >>> for commit in commit_history.commits:
            ...     print(f"{commit.short_id}: {commit.title}")
        r   is_relative_pathr   rn   Invalid repo_id:  !z/api/v1/s/z/commitsr   )r   r}  r~  rm  r   r{   Codez%Failed to get repository commits for : N)datasets.utils.file_utilsr  r  r   r   rk   rO   ry   r   r|   r   r   r{   r>   r   r?   r   r   rd   Zfrom_api_responserequests
exceptionsRequestExceptionr   r)  )r   r   r   r   r=  r>  rk   r  _owner_dataset_nameZcommits_urlrm  r   r   rs  er   r   r   r   !  s8   
zHubApi.list_repo_commitsr   )r   rw  r   r=  r>  rk   c                C   s   ddl m} ||r|ddkr|d\}	}
ntd| d| j|
|	|d\}}|s0| j}| d| d	}|||r>d
nd||d}t }| j	j
|||d}| }t||| |d d S )aZ  
        Get the dataset files.

        Args:
            repo_id (str): The repository id, in the format of `namespace/dataset_name`.
            revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
            root_path (str): The root path to list. Defaults to '/'.
            recursive (bool): Whether to list recursively. Defaults to True.
            page_number (int): The page number for pagination. Defaults to 1.
            page_size (int): The number of items per page. Defaults to 100.
            endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

        Returns:
            List: The response containing the dataset repository tree information.
                e.g. [{'CommitId': None, 'CommitMessage': '...', 'Size': 0, 'Type': 'tree'}, ...]
        r   r  r   rn   r  r  rx  r   r{  ry  rz  r|  r  rp  r`  )r  r  r  r   r   ru  rk   ry   r   r|   r   r   r:   )r   r   r   rw  r   r=  r>  rk   r  r  r  r  rt  rr  rm  r   r   rs  r   r   r   r   _  s*   

zHubApi.get_dataset_filesc                 C   sx   t  }|s	| j}|r| d| d| }n| d| }| jj||| | jd}t| | }t	||| |t
 S )ac  
        Get the dataset information.

        Args:
            dataset_id (str): The dataset id.
            revision (Optional[str]): The revision of the dataset.
            endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

        Returns:
            dict: The dataset information.
        r   r   r   )ry   r   rk   r|   r   r   r{   r>   r   r:   r   )r   r   r   rk   r   r   r   rs  r   r   r   r     s   zHubApi.get_datasetc                 C   s   |s| j }| d| d| }t }| jj||| | jd}| }	t||	| |	d }
|
du r@t	d| d| d| d	|
d
 }
|
S )z( Get the meta file-list of the dataset. r   z/repo/tree?Revision=r   rp  Nz'The modelscope dataset [dataset_name = z, namespace = z, version = z] dose not existr`  )
rk   ry   r   r|   r   r   r{   r   r:   r7   )r   rh  ri  r   r   rk   rr  r   r   rs  	file_listr   r   r   get_dataset_meta_file_list  s&   
z!HubApi.get_dataset_meta_file_listrt  meta_cache_dirc                 C   sX   t j|t|  tjj }t|d}|d W d   dS 1 s%w   Y  dS )z
        Dump the data_type as a local file, in order to get the dataset
         formation without calling the datahub.
        More details, please refer to the class
        `modelscope.utils.constant.DatasetFormations`.
        r  z3*** Automatically-generated file, do not modify ***N)	r   r   r	  r)  rV   Zformation_mark_extvaluer  write)rt  r  Zdataset_type_file_pathfpr   r   r   dump_datatype_file  s   "zHubApi.dump_datatype_filer  c              
   C   s(  t t}t|}	t|	 }
t }tj||d |s| j}|D ]q}|d }t	j
|d }||
v r| d| d| d| d| 	}| jj||d}t| t	j
||}t	j
|rmtd	| d
| d || | qt|d}||j W d    n1 sw   Y  || | q||	fS )N)rt  r  r   r   r   r   /repo?Revision=
&FilePath=ro  zReusing dataset z's python file ()wb)r   listrV   rW   ry   r   rj   r  rk   r   r   splitextr|   r   r>   r	  r%  r   r   rd  r  r  content)r   rh  ri  r   r  rt  r  rk   Zlocal_pathsZdataset_formationZdataset_meta_formatr   	file_info	file_path	extensionrr  r   Z
local_pathr   r   r   r   "get_dataset_meta_files_local_paths  s>   z)HubApi.get_dataset_meta_files_local_paths   c                    s  ddl }ddlm} ddl}tj||| jdd	 }|t
jkr.tj|r.t| tj|r>td|  |S t }td tj| |dd	}t|jd
d}	||	dd}
 fdd}d}t|dQ}||D ]D}|
t| | drdd |D }t|dkrqp|dkrd}nd}||}|j|d|dd |d7 }qp|D ]	}||d  qqpW d   n1 sw   Y  |
  |S )zO
        Fetch the meta-data files from the url, e.g. csv/jsonl files.
        r   Nr   zUTF-8encodingzReusing cached meta-data file: zLoading meta-data file ...T)r   streamzcontent-length)rp   Zdynamic_ncolsc                 3   sH    g }|   D ]}|d}|| t| kr|V  g }q|V  d S )Nutf-8)
iter_linesdecoderd  r[  )rs  Z
chunk_datar?  
chunk_sizer   r   	get_chunk  s   


z3HubApi.fetch_meta_files_from_url.<locals>.get_chunkaZjsonlc                 S   s   g | ]}|  rt|qS r   )r   r   loads)r  liner   r   r   r!     s    z4HubApi.fetch_meta_files_from_url.<locals>.<listcomp>F\)indexheader
escapecharrn   
)hashlib	tqdm.autor   Zpandasr   r   r	  md5encode	hexdigestrY   ZFORCE_REDOWNLOADr%  r+  r   r  ry   r   r  r   rV  r{   r  r   r[  r   Z	DataFrameZto_csvr  close)urlZout_pathr  moder  r   pdr   response
total_sizeprogressr  Ziter_numr   chunkZwith_headerZchunk_dfr  r   r  r   fetch_meta_files_from_url  sH    





z HubApi.fetch_meta_files_from_url	file_nameviewextension_filterc           
      C   sR   |r|r|s
t dd|||d}t|}|s| j}| d| d| d| }	|	S )Nz:Args (file_name, dataset_name, namespace) cannot be empty!ZSDK)ZSourcerH  FilePathZViewr   r   /repo?)r   r   rk   )
r   r  rh  ri  r   r  r  rk   rm  file_urlr   r   r   get_dataset_file_url2  s   
zHubApi.get_dataset_file_urlc              	   C   sF   |s| j }|r!tj|d tv r!| d| d| d| d| 	}|S )Nr   r   r   r  r  )rk   r   r   r  rQ   )r   r  rh  ri  r   rk   r   r   r   get_dataset_file_url_originR  s   z"HubApi.get_dataset_file_url_originc                 C   s.   |s| j }| d| d| d| }| |S )Nr   r   /ststoken?Revision=)rk   datahub_remote_call)r   rh  ri  r   rk   rr  r   r   r   get_dataset_access_config`  s   
z HubApi.get_dataset_access_configcheck_cookiec           
      C   sp   |s| j }| d| d| d| }|r| jdd}nt }| jj||| | jd}| }	t	|	 |	d S )Nr   r   r  T)rB  r  r   r{   rp  )
rk   rD  ry   r   r|   r   r   r{   r   r?   )
r   rh  ri  r  r   rk   rr  r   r   rs  r   r   r   !get_dataset_access_config_sessionl  s    
z(HubApi.get_dataset_access_config_sessionversionc           
      C   s   t jtjd}|stdtj | d}tjt	
 }t||d}t|d}| jj|||| | jdd}| }	|	d d	krJtd
|	 |	d S )z.
        Get virgo dataset meta info.
        r   z"Virgo endpoint is not set in env: z/data/set/download)Z	dataSetIdZdataSetVersion)r?  i  )r  r   r   r{   rx   r  r   zFailed to get virgo dataset: r?  )r   r   r   r]   Zenv_virgo_endpointRuntimeErrorr  utilsdict_from_cookiejarry   r   dictr|   r   r   r{   r   )
r   r   r  Zvirgo_endpointZvirgo_dataset_urlr   r   r?  r   rs  r   r   r   get_virgo_meta  s,   

zHubApi.get_virgo_metazip_file_namec                 C   s   |s| j }| d| d| }t }| jj||| | jd}| }	t|	 |	d }
t	|
d }| d| }| jj||| | jd}| }t| |d }|d d | d | d | }||d	< |S )
Nr   r   r  rp  r4   r  z	-unzippedr   ZDir)
rk   ry   r   r|   r   r   r{   r   r?   r5   )r   rh  ri  r   r  rk   rr  r   r   rs  r?  r   Zdatahub_sts_urlZr_stsZresp_stsZdata_stsZfile_dirr   r   r   &get_dataset_access_config_for_unzipped  s*   



 z-HubApi.get_dataset_access_config_for_unzippedc                 C   sl   |s| j }| d| d| d| d| d| d| }t }	| jj||	dd}
|
 }
t|
 |
d	 }
|
S )
Nr   r   z/oss/tree/?MaxLimit=
&Revision=z&Recursive=z&FilterDir=i  )r  r   rx   rp  )rk   ry   r   r|   r   r   r?   )r   rh  ri  	max_limitZis_recursiveZis_filter_dirr   rk   r  r   rs  r   r   r   list_oss_dataset_objects  s"   zHubApi.list_oss_dataset_objectsobject_namec           	   	   C   v   |r|r|r|st d|s| j}| d| d| d| d| 	}t }| jj||d}| }t| |d }|S )NArgs cannot be empty!r   r   z
/oss?Path=r  r  r   Messager   rk   ry   r   r|   r   r   r?   	r   r  rh  ri  r   rk   r  r   rs  r   r   r   delete_oss_dataset_object  s    z HubApi.delete_oss_dataset_objectc           	   	   C   r  )Nr  r   r   z/oss/prefix?Prefix=z/&Revision=r  r  r  r  r   r   r   delete_oss_dataset_dir  s   zHubApi.delete_oss_dataset_dirc                 C   s>   t  }| jj||dt  id}| }t||| |d S )Nrl   r   rp  )ry   r   r|   r   rz   r   r:   )r   r  r   r   rs  r   r   r   r    s   
zHubApi.datahub_remote_calluse_streamingc              
   C   s.  t ddk}|s| j}|r|r|s|szbt }| d| d| d}| jj||| | jd}t	| t
jj}	d}
tt jv rEt jt }	tt jv rOt jt }
| d| d| d|	 d	|
 	}| jj||| | jd}| }t| W d S  ty } zt| W Y d }~d S d }~ww d S d S d S d S )
NZCI_TESTry  r   r   z/download/increaser   r   z/download/uv/z?user=)r   getenvrk   ry   r   r|   r   r   r{   r>   rX   ZLOCALr  r    r   r!   r   r?   r   r   r   )r   rh  ri  r  rk   Z
is_ci_testr   Zdownload_count_urlZdownload_count_respZchannel	user_nameZdownload_uv_urlZdownload_uv_respr  r   r   r   dataset_download_statistics  s@   







z"HubApi.dataset_download_statisticsc                 C   s   t tt ji|S r  )r%   r)  ra  rb  rc  )r   r{   r   r   r   r     s   zHubApi.builder_headersc                 C   s.   | d\}}|s| j}| d| d| dS )Nr   r   r  )r   rk   )r   r   rk   Z
_namespacer  r   r   r   get_file_base_path  s   zHubApi.get_file_base_path)	r   r   r   r   r   rk   exist_okcreate_default_configr   r  r  c       	         K   s  |st d|s| j}| j||d | j||||d}|r>|r6| d| d| }td| d|  |S t d| d|d}t|d	krMt d
|\}}|tkrdd t	j
 D }|| }|du rrt d| d| j|||||
d}|	rt 4}ddlm} |||}dddd}|d}|si }i ||}t|dt|gdd W d   n1 sw   Y  td| ddd |S |tkrdd tj
 D }|| }|du rt d| d| j|||||d}td| ddd |S t d | d!t )"a%  
        Create a repository on the ModelScope Hub.

        Args:
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            token (Union[str, bool, None]): The access token.
            visibility (Optional[str]): The visibility of the repo,
                could be `public`, `private`, `internal`, default to `public`.
            repo_type (Optional[str]): The repo type, default to `model`.
            chinese_name (Optional[str]): The Chinese name of the repo.
            license (Optional[str]): The license of the repo, default to `apache-2.0`.
            endpoint (Optional[str]): The endpoint to use.
                In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
            exist_ok (Optional[bool]): If the repo exists, whether to return the repo url directly.
            create_default_config (Optional[bool]): If True, create a default configuration file in the model repo.
            **kwargs: The additional arguments.

        Returns:
            str: The repo url.
        Repo id cannot be empty!)r   rk   )r   rk   r   r   r  r   z already exists, got repo url: z already exists!rm   zBInvalid repo id, should be in the format of `owner_name/repo_name`c                 S       i | ]\}}| d s||qS __
startswithr  kvr   r   r   
<dictcomp>Z       z&HubApi.create_repo.<locals>.<dictcomp>NzInvalid visibility: z9, supported visibilities: `public`, `private`, `internal`)r   r   r   r   r   r   rC   Zpytorchztext-generationT)r  r  Zallow_remoteconfig_jsonzconfiguration.json)Zignore_push_errorz"New model created successfully at r   flushc                 S   r  r  r  r  r   r   r   r  z  r  )rh  ri  r   r   r   z$New dataset created successfully at Invalid repo type: , supported repos: )r   rk   r   r   r   r   r   r[  rS   r3   __dict__itemsr   upperr   tempfileTemporaryDirectorymodelscope.hub.repositoryrD   rF   r   dumpsprintrR   r1   rl  rT   )r   r   r   r   r   r   r   rk   r  r  r   kwargsr   repo_urlZrepo_id_listri  	repo_nameZvisibilitiesZtemp_cache_dirrD   r6  Zdefault_configr  configr   r   r   create_repo   s~   $




zHubApi.create_repo      )commit_descriptionr   r   r   rk   rv   rx   
operationsr  rv   rx   c                C   s|  |st d|s| j}|tvrt d| dt | d| d| d| }|p,d| }|p0d}| j|d	d
}| j||d}d}t|	D ]}z|dkrZtd|d  d| d tj	|| 
| jt|||
d}|jdkrz| }W n tjy   |j}Y nw d|j d| d| }d|j  krdk rn ntd|d  d|  n(t d| | }|di dd}td|  t||||dW   S W nP tjjy } z|}td|d  dt|  W Y d}~n0d}~w ty$ } z|}td |d  dt|  ||	d kr W Y d}~nd}~ww ||	d k r1td qEtjd!|	 d"| )#a  
        Create a commit on the ModelScope Hub with retry mechanism.

        Args:
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            operations (Iterable[CommitOperation]): The commit operations.
            commit_message (str): The commit message.
            commit_description (Optional[str]): The commit description.
            token (str): The access token. If None, will use the cookies from the local cache.
                See `https://modelscope.cn/my/myaccesstoken` to get your token.
            repo_type (Optional[str]): The repo type, should be `model` or `dataset`. Defaults to `model`.
            revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
            endpoint (Optional[str]): The endpoint to use.
                In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
            max_retries (int): Number of max retry attempts (default: 3).
            timeout (int): Timeout for each request in seconds (default: 180).

        Returns:
            CommitInfo: The commit info.

        Raises:
            requests.exceptions.RequestException: If all retry attempts fail.
        r  r  r  /api/v1/repos/r  z/commit/z
Commit to r   Tr   )r  r  Nr   zAttempt rn   z to create commit for z...)r{   r?  r   rx   r  zHTTP z error from r  ro   iX  zServer error on attempt zClient request failed: rp  oidzCommit succeeded: )Z
commit_urlr  r  r  zRequest failed on attempt zUnexpected error on attempt zFailed to create commit after z attempts. Last error: )r   rk   rT   r   _prepare_commit_payloadranger   r  r  r   r   r{   r   r  r   JSONDecodeErrortextr   r   re   r  r  r)  r   r   timesleep)r   r   r  r  r  r   r   r   rk   rv   rx   r  r   payloadZlast_exceptionattemptr  Zerror_detail	error_msgrs  r  r  r   r   r   create_commit  s   %


*
zHubApi.create_commitz[Uploading])r   r   r  r  buffer_size_mb	tqdm_descdisable_tqdmr   path_or_fileobjpath_in_repor  r  r  c             	   C   s  |t vrtd| dt  |std| j|dd t|ttfr5tjtj	|}|p3tj
|}n|s;tdt|tjrE| }| j| | jj|g|d |durZ|nd	| d
}|dkrhtdt||d}|d }|d }| j|||| jddd | j||||||
|	d}t|||d}| j||rdnd|_|d |_|g}td| ddd | j|||||||d}|S )a  
        Upload a file to the ModelScope Hub.

        Args:
            path_or_fileobj (Union[str, Path, bytes, BinaryIO]):
                The local file path or file-like object (BinaryIO) or bytes to upload.
            path_in_repo (str): The path in the repo to upload to.
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
                See `https://modelscope.cn/my/myaccesstoken` to get your token.
            repo_type (Optional[str]): The repo type, default to `model`.
            commit_message (Optional[str]): The commit message.
            commit_description (Optional[str]): The commit description.
            buffer_size_mb (Optional[int]): The buffer size in MB for reading the file. Default to 1MB.
            tqdm_desc (Optional[str]): The description for the tqdm progress bar. Default to '[Uploading]'.
            disable_tqdm (Optional[bool]): Whether to disable the tqdm progress bar. Default to False.
            revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

        Returns:
            CommitInfo: The commit info.

        Examples:
            >>> from modelscope.hub.api import HubApi
            >>> api = HubApi()
            >>> commit_info = api.upload_file(
            ...     path_or_fileobj='/path/to/your/file.txt',
            ...     path_in_repo='optional/path/in/repo/file.txt',
            ...     repo_id='your-namespace/your-repo-name',
            ...     commit_message='Upload file.txt to ModelScope hub'
            ... )
            >>> print(commit_info)
        r  r  z$Path or file object cannot be empty!Tr   z#Arg `path_in_repo` cannot be empty!file_path_listr   NzUpload z to ModelScope hubr   z4Buffer size: `buffer_size_mb` must be greater than 0)file_path_or_objr  	file_size	file_hashFr   r   r   rk   r  r  r   r   sha256sizer?  r  r  r  r  file_hash_infolfsnormalis_uploadedzCommitting file to z ...r  r   r  r  r  r   r   r   )rT   r   r   r(  r)  r   r   r   abspathr   basenameioBufferedIOBaserq   r   
check_filecheck_normal_filesr^   r	  rk   _upload_blobrg   is_lfs_upload_mode_is_uploadedr  r  )r   r  r  r   r   r   r  r  r  r  r  r   hash_info_dr"  r#  
upload_resZadd_operationr  commit_infor   r   r   upload_file  s|   0

zHubApi.upload_file)	r  r  r  r   r   allow_patternsignore_patternsmax_workersr   folder_pathr<  r=  r>  c             
      s   st d|du rt dtvrt d dt j|dd |r'|nd}|	r-|	nd}	|	du r6g }	nt|	tr>|	g}	|	t7 }	|durH|nd  d	}|pQd
}td j||||	d}t	|dkrnt d| dtdt	| d j
jdd |D d j |jddd t|
dd fdd}|| |||ddd}g }|D ]-}|d }|d }|d }|d }t|||d }j
|rd!nd"|_||_|| qt	|dkrt d| dtdkrtnt	|}t	|d | d }td#t	| d$| d%| d&dd' g }tt|d(|d)D ]-}||| |d |  }| d*|d  d+| d,}j |||||d-}|| qt	|dkrV|d S |S ).a  
        Upload a folder to the ModelScope Hub.

        Args:
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            folder_path (Union[str, Path, List[str], List[Path]]): The folder path or list of file paths to upload.
            path_in_repo (Optional[str]): The path in the repo to upload to.
            commit_message (Optional[str]): The commit message.
            commit_description (Optional[str]): The commit description.
            token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
                See `https://modelscope.cn/my/myaccesstoken` to get your token.
            repo_type (Optional[str]): The repo type, default to `model`.
            allow_patterns (Optional[Union[List[str], str]]): The patterns to allow.
            ignore_patterns (Optional[Union[List[str], str]]): The patterns to ignore.
            max_workers (int): The maximum number of workers to use for uploading files concurrently.
                Defaults to `DEFAULT_MAX_WORKERS`.
            revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

        Returns:
            Union[CommitInfo, List[CommitInfo]]:
                The commit info or list of commit infos if multiple batches are committed.

        Examples:
            >>> from modelscope.hub.api import HubApi
            >>> api = HubApi()
            >>> commit_info = api.upload_folder(
            ...     repo_id='your-namespace/your-repo-name',
            ...     folder_path='/path/to/your/folder',
            ...     path_in_repo='optional/path/in/repo',
            ...     commit_message='Upload my folder',
            ...     token='your-access-token'
            ... )
            >>> print(commit_info.commit_url)
        z"The arg `repo_id` cannot be empty!Nz%The arg `folder_path` cannot be None!r  r  Tr   z
Upload to z on ModelScope hubzUploading fileszPreparing files to upload ...)folder_path_or_filesr  r<  r=  r   z"No files to upload in the folder: r  z	Checking z files to upload ...c                 S   s   g | ]\}}|qS r   r   )r  r   rR  r   r   r   r!    rK  z(HubApi.upload_folder.<locals>.<listcomp>r  Fr$  )r>  r  c              	      sX   | \}}t |d}|d }|d }j ||||tkd| d d}|||d |dS )	N)r!  r"  r#  z[Uploading ]r%  r,  )file_path_in_repor  r,  r)  )r^   r4  r*   )Z	item_pairr  rB  r  r8  r"  r#  r9  r   r   r   r   r   _upload_items  s(   
z+HubApi.upload_folder.<locals>._upload_itemsrn   )r   r   r   r  r  r  r  rB  r  r,  r)  r(  r*  r+  zCommitting z
 files in z batch(es) of size r   r  z[Committing batches] )descrp   z (batch r   r  r-  )r   rT   r   r(  r)  rb   r   r  _prepare_upload_folderr[  r   r3  r	  rk   ri   rg   r5  r6  r7  rd  r+   r  r   r  r  )r   r   r?  r  r  r  r   r   r<  r=  r>  r   prepared_repo_objectsrD  Zuploaded_items_listr  Zitem_dZprepared_path_in_repoZprepared_file_pathr,  r)  optZcommit_batch_sizeZnum_batchesZcommit_infosiZbatch_operationsZbatch_commit_messager:  r   rC  r   upload_folder  s   1


	zHubApi.upload_folder)r  r  r  r&  r'  r?  c             	   C   s  t d dd d d}	||dg}
| j|||
d}t|dkr |d nd }|d u r9td|d d  d	 d
|	d< |	S t }|rCt |nd }|d u rMtd| j	dd|d  i | 
| j}|d d fdd}t|dd
||d`}t|ttfrt|d}tj|d ||||d}W d    n1 sw   Y  n.t|trtj|d ||t||d}nt|tjrtj|d ||||d}ntdW d    n1 sw   Y  t|d | }t|d |d |	d< |d |	d< |d |	d< |	S )NF)r  r,  r   
status_msg)r  r'  )r   r   objectsrn   r   zBlob    z  has already uploaded, reuse it.Tr,  r   Cookiezm_session_id=r   r  c                 s   s,    	 |  |}|sdS |t| |V  q)z8Lazy function (generator) to read a file piece by piece.TN)rq   r   r[  )Zfile_objectpbarr  Zckr   r   r   read_in_chunksi	  s   
z+HubApi._upload_blob.<locals>.read_in_chunksB)rp   unitZ
unit_scalerE  disablerbr  )r{   r?  zInvalid data type to uploadZrspr  r   r  rK  )r  _validate_blobr[  r   rZ  ry   r   r   r{   r   r   r   r(  r)  r   r  r  r@  bytesr0  BytesIOr1  r>   r   r?   )r   r   r   r&  r'  r?  r  r  r  Zres_drL  upload_objectsZupload_objectr   r{   rP  rO  r   r  rs  r   r   r   r4  >	  s~   	

!
zHubApi._upload_blobr   rL  c                C   s   |s| j }| d| d| d}d|d}t }|du r!tdtj|| | jt	||d}t
|d	 | }	t|	d	 g }
|	d
 d }|D ]}|
|d d d |d d qI|
S )aC  
        Check the blob has already uploaded.
        True -- uploaded; False -- not uploaded.

        Args:
            repo_id (str): The repo id ModelScope.
            repo_type (str): The repo type. `dataset`, `model`, etc.
            objects (List[Dict[str, Any]]): The objects to check.
                oid (str): The sha256 hash value.
                size (int): The size of the blob.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            List[Dict[str, Any]]: The result of the check.
        r  r  z/info/lfs/objects/batchZupload)	operationrL  Nr   )r{   r?  r   rU  rp  rL  actionshrefr  )r  r  )rk   ry   r   r   r  r   r   r{   r   r  r>   r?   rd  )r   r   r   rL  rk   r  r  r   r  rs  rY  Zresp_objectsobjr   r   r   rV  	  s6   


zHubApi._validate_blobr@  c           	         s6  d  d }t |trtj|d r|}ntdtj|r"|g}n| |d u rQ| j  t 	 
    sBtd  d fddt dD ni |D ]}tj|rk| j| |tj|< qUttj ||d}|r|d	 d	nd
fdd|D }tdt| d |S )Nr   z0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directoryc                    s$   i | ]}|  r|  |qS r   )is_filerelative_toas_posix)r  r   )r?  r   r   r  	  s    z1HubApi._prepare_upload_folder.<locals>.<dictcomp>z**/*)r<  r=  r   r   c                    s    g | ]} | t | fqS r   )r)  )r  relpath)prefixrelpath_to_abspathr   r   r!  
  s    z1HubApi._prepare_upload_folder.<locals>.<listcomp>z	Prepared z files for upload.)r(  r  r   r   r&  r   r   check_folderr   r   resolveis_dirsortedglobr2  r/  rh   Zfilter_repo_objectskeysr   r   r  r[  )	r   r@  r  r<  r=  Z
files_pathr   Zfiltered_repo_objectsrG  r   )r?  rb  rc  r   rF  	  sD   


zHubApi._prepare_upload_folderc              	   C   s  |g d}d}| D ]t}t |tr"|jr"td|j d |d7 }q	t |trJ|jdkrJ|jr1dnd|jd|jj	d	|
  d
d}|d | q	t |trp|jdkrp|jrYdnd|jd|jj	|jjd	d	d}|d | q	td| dt|dd |dkrtd| d |S )zN
        Prepare the commit payload to be sent to the ModelScope hub.
        )r  r[  r   zSkipping file 'z(' in commit (ignored by gitignore file).rn   r+  r   creater   base64)actionr   typer'  r&  r  r  r[  r*  z(Unknown operation to commit. Operation: z. Upload mode: r6  NzSkipped z/ file(s) in commit (ignored by gitignore file).)r(  rg   Z_should_ignorer   rZ  r  r6  r7  Zupload_infor'  Z
b64contentr  rd  r&  r   r   r  )r  r  r  Znb_ignored_filesrZ  Zcommit_actionr   r   r   r  
  sJ   

		
zHubApi._prepare_commit_payload皙?internal_timeoutc                 C   s~   dt dtfdd}| j d}|||d}d}|dur=| }d	|v r*|d	 d
 }nd}|r=|||d}|dur=|j }|S )a  
        Get the internal acceleration domain.

        Args:
            internal_timeout (float): The timeout for the request. Default to 0.2s

        Returns:
            str: The internal acceleration domain. e.g. `cn-hangzhou`, `cn-zhangjiakou`
        r  rx   c                 S   s:   zt j| |d}|  W |S  t jjy   d }Y |S w )Nrw   )r  r   raise_for_statusr  r  )r  rx   r  r   r   r   send_request\
  s   
z>HubApi._get_internal_acceleration_domain.<locals>.send_requestz&/api/v1/repos/internalAccelerationInfo)r  rx   r   Nrp  ZInternalRegionQueryAddressrw   )r)  floatrk   r   r  r   )r   ro  rq  Zinternal_urlZinternal_info_responseZ	region_idZ
query_addrZdomain_responser   r   r   !_get_internal_acceleration_domainQ
  s   	
z(HubApi._get_internal_acceleration_domaindelete_patternsc                C   s  |t vrtd| |stdt|tr|g}t }|s"| j}|du r*td| | j}|t	krH| j
||p:td||d}dd |D }	nc|tkrg }	d	}
d
}	 z| j||pZtd|
||d}W n ty } ztd| dt|  W Y d}~n#d}~ww |D ]}|d dkr|	|d  qt||k rn|
d	7 }
qSn
td| dt  g }|	D ]}|D ]}t||r||  nqqg g }}|D ]}za|t	kr|d\}}| d| d| d}|pt|d}n&|tkr|d\}}| d| d| d}d|i}n
td| dt  | jj||||d}t| | }t| || W q tyS } z|| td| dt|  W Y d}~qd}~ww ||t|dS )a=  
        Delete files in batch using glob (wildcard) patterns, e.g. '*.py', 'data/*.csv', 'foo*', etc.

        Example:
            # Delete all Python and Markdown files in a model repo
            api.delete_files(
                repo_id='your_username/your_model',
                repo_type=REPO_TYPE_MODEL,
                delete_patterns=['*.py', '*.md']
            )

            # Delete all CSV files in the data/ directory of a dataset repo
            api.delete_files(
                repo_id='your_username/your_dataset',
                repo_type=REPO_TYPE_DATASET,
                delete_patterns='data/*.csv'
            )

        Args:
            repo_id (str): 'owner/repo_name' or 'owner/dataset_name', e.g. 'Koko/my_model'
            repo_type (str): REPO_TYPE_MODEL or REPO_TYPE_DATASET
            delete_patterns (str or List[str]): List of glob patterns, e.g. '*.py', 'data/*.csv', 'foo*'
            revision (str, optional): Branch or tag name
            endpoint (str, optional): API endpoint
        Returns:
            dict: Deletion result
        zUnsupported repo_type: zdelete_patterns cannot be emptyNr   T)r   r   rk   rB  c                 S   rF  r   r   )r  r   r   r   r   r!  
  rK  z'HubApi.delete_files.<locals>.<listcomp>rn   rv  )r   r   r   r=  r>  rk   zGet dataset: z file list failed, message: rq  treer   r  r   r   z/file)rH  r  r   z/repor  r  zFailed to delete r  )deleted_filesfailed_filesZtotal_files)rT   r   r(  r)  ry   r   rk   r   r{   rS   r   rN   rR   r   rM   r   r   r   rd  r[  fnmatchr   r|   r   r>   r   r?   )r   r   r   rt  r   rk   r   r{   re  
file_pathsr=  r>  Zdataset_filesr  Zfile_info_dZ	to_deleter   Zdelete_patternrv  rw  ownerr  r  rm  rh  r   rs  r   r   r   delete_filesx
  s   "





&zHubApi.delete_files)Fr   )NNNr  )rn   r<  N)NF)NFN)FN)Trn   rv  N)rn   )rn  )j__name__
__module____qualname____doc__r   r   r   r)  r   r   rC  r   r   r3   ZPUBLICr2   Z	APACHE_V2rV  r   r   r   r   rN   r   r   rB   r   rA   r   rS   r   r   r   r  staticmethodr  rO   r   r;  r  rA  r   rD  rN  rL  rS  r\  r]  r   rY  r^  r   rg  r1   rl  rn  r  ru  r  r   r   r   r  r  r  r  rY   ZREUSE_DATASET_IF_EXISTSr  rM   r  r  r  r  r  r  r  r  r  r  r  r   r  r4   rE   r	  r   rf   re   r  r   rW  r	   r;  r   rJ  r4  r
   r   rV  tuplerF  r  rr  rs  r{  r   r   r   r   rj   a   s   
!
2
j
U
.
-
#
$

'
:
	

 
(



"
G


 



;

(
$
A
8
"

#<
%


"




 	

s	

~
	

 
	

 @	

e
?
9
?,
rj   c                   @   s   e Zd ZeeZdZdZdZdZ	dZ
edd Zedefdd	Zed
d Zedd ZedefddZededefddZedeeef fddZedee fddZeddeeedf defddZdS )ry   r   Z	git_tokenuserr|   Fc                   C   s   t jtjdd d S )NT)r  )r   makedirsry   path_credentialr   r   r   r   make_sure_credential_path_exist  s   z0ModelScopeConfig.make_sure_credential_path_existc                 C   sR   t   ttjt jt jd}t	| | W d    d S 1 s"w   Y  d S )Nzwb+)
ry   r  r  r   r   r	  r  COOKIES_FILE_NAMEpickler  )r   r   r   r   r   r     s   
"zModelScopeConfig.save_cookiesc                  C   s   t jtjtj} t j| rMt| d1}t	|}|D ]}|j
dkr;| r;tjs;dt_td  W d    d S q|W  d    S 1 sHw   Y  d S )NrT  r   TzKNot logged-in, you can login for uploadingor accessing controlled entities.)r   r   r	  ry   r  r  r%  r  r  loadr   
is_expiredcookie_expired_warningr   r  )Zcookies_pathr   r   cookier   r   r   r     s$   


	 
zModelScopeConfig.get_cookiesc                  C   s   t jtjtj} d}t j| r2t| d}t|	 
 dd}|W  d    S 1 s-w   Y  |dks<t|dkrbtt j}t  t| d}|| W d    |S 1 s]w   Y  |S )Nr   rT  r  r      w+)r   r   r	  ry   r  USER_SESSION_ID_FILE_NAMEr%  r  r)  readliner   r[  ra  rb  rc  r  r  )Zsession_pathZ
session_idr   Zwfr   r   r   get_user_session_id   s$   
 
z$ModelScopeConfig.get_user_session_idr   c                 C   sP   t   ttjt jt jd}||  W d    d S 1 s!w   Y  d S )Nr  )	ry   r  r  r   r   r	  r  GIT_TOKEN_FILE_NAMEr  r   r   r   r   r   r   1  s   
"zModelScopeConfig.save_tokenr  
user_emailc                 C   sX   t   ttjt jt jd}|d| |f  W d    d S 1 s%w   Y  d S )Nr  z%s:%s)	ry   r  r  r   r   r	  r  USER_INFO_FILE_NAMEr  )r  r  r   r   r   r   r   9  s   
"zModelScopeConfig.save_user_infor   c                  C   s~   z4t tjtjtjddd} |  }|dd |dd fW  d    W S 1 s-w   Y  W dS  t	y>   Y dS w )Nr   r  r  :r   rn   r   )
r  r   r   r	  ry   r  r  rq   r   FileNotFoundError)r   r  r   r   r   get_user_infoA  s"   
$	zModelScopeConfig.get_user_infoc                  C   sh   d} z't tjtjtjddd}| } W d   W | S 1 s"w   Y  W | S  ty3   Y | S w )z
        Get token or None if not existent.

        Returns:
            `str` or `None`: The token, `None` if it doesn't exist.

        Nr   r  r  )	r  r   r   r	  ry   r  r  rq   r  r  r   r   r   	get_tokenO  s&   	

zModelScopeConfig.get_tokenN
user_agentc                 C   s   d}t tjv rtjt  }d}ttjv rtjt }ddlm} d|t t	 t t
 ||f }t| trH|dddd |  D  7 }|S t| trS|d|  7 }|S )	a  Formats a user-agent string with basic info about a request.

        Args:
            user_agent (`str`, `dict`, *optional*):
                The user agent info in the form of a dictionary or a single string.

        Returns:
            The formatted user-agent string.
        Zcustomunknownr   )__version__zSmodelscope/%s; python/%s; session_id/%s; platform/%s; processor/%s; env/%s; user/%sz; c                 s   s"    | ]\}}| d | V  qdS )r   Nr   r  r   r   r   	<genexpr>  s     z2ModelScopeConfig.get_user_agent.<locals>.<genexpr>)r    r   r   r!   Z
modelscoper  platformpython_versionry   r  	processorr(  r  r	  r  r)  )r  envr  r  Zuar   r   r   rz   d  s,   




	 
zModelScopeConfig.get_user_agentr  )r|  r}  r~  r   r"   r  r  r  r  r  r  r  r  r   r   r   r  r)  r   r   r   r  r   r  r   r
   rz   r   r   r   r   ry   
  s2    


$ry   c                   @   s   e Zd ZdZeeeeefde	de	de	de	de	f
ddZ
dddZdeeef fddZdeeeeef ded	efddZdeeeef  ded	d
fddZd
S )r   a  
    Check the files and folders to be uploaded.

    Args:
        max_file_count (int): The maximum number of files to be uploaded. Default to `UPLOAD_MAX_FILE_COUNT`.
        max_file_count_in_dir (int): The maximum number of files in a directory.
            Default to `UPLOAD_MAX_FILE_COUNT_IN_DIR`.
        max_file_size (int): The maximum size of a single file in bytes. Default to `UPLOAD_MAX_FILE_SIZE`.
        size_threshold_to_enforce_lfs (int): The size threshold to enforce LFS in bytes.
            Files larger than this size will be enforced to be uploaded via LFS.
            Default to `UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS`.
        normal_file_size_total_limit (int): The total size limit of normal files in bytes.
            Default to `UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT`.

    Examples:
        >>> from modelscope.hub.api import UploadingCheck
        >>> upload_checker = UploadingCheck()
        >>> upload_checker.check_file('/path/to/your/file.txt')
        >>> upload_checker.check_folder('/path/to/your/folder')
        >>> is_lfs = upload_checker.is_lfs('/path/to/your/file.txt', repo_type='model')
        >>> print(f'Is LFS: {is_lfs}')
    max_file_countmax_file_count_in_dirmax_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitc                 C   s"   || _ || _|| _|| _|| _d S r  )r  r  r  r  r  )r   r  r  r  r  r  r   r   r   r     s
   
zUploadingCheck.__init__r   Nc                 C   sl   t |ttfrtj|std| dt|}|| jkr4t	
d| jd  dt|d d d dS dS )	a  
        Check a single file to be uploaded.

        Args:
            file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.

        Raises:
            ValueError: If the file does not exist or exceeds the size limit.
        File  does not existzFile exceeds size limit:    @z	 GB, got     GBN)r(  r)  r   r   r   r%  r   r_   r  r   r   rW  )r   r!  r"  r   r   r   r2    s   

zUploadingCheck.check_filer?  c                 C   s  d}d}t |trt|}| D ][}| r>|d7 }t|}|| jkr=td| d| jd  ddt	|d d d q|
 rl|d7 }| |\}}|| | jkrdtd	| d
||  d| j ||7 }||7 }q|| jkr}td| d| j ||fS )z
        Check a folder to be uploaded.

        Args:
            folder_path (Union[str, Path]): The folder path to be checked.

        Raises:
            ValueError: If the folder does not exist or exceeds the file count limit.
        r   rn   r  z exceeds size limit: r  r  zgot r  z
Directory z
 contains z items and exceeds limit: zTotal file count z and exceeds limit: )r(  r)  r   iterdirr^  r_   r  r   r   rW  rf  rd  r  r   r  )r   r?  Z
file_countZ	dir_countrR  Z	item_sizeZsub_file_countZsub_dir_countr   r   r   rd    s4   



zUploadingCheck.check_folderr!  r   c                 C   s   d}t |ttfr;t|}| std| d|tkr%|jtvr$d}n|tkr1|jt	vr0d}n
td| dt
 t|}|| jkpE|S )aj  
        Check if a file should be uploaded via LFS.

        Args:
            file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.
            repo_type (str): The repo type, either `model` or `dataset`.

        Returns:
            bool: True if the file should be uploaded via LFS, False otherwise.
        Tr  r  Fr  r  )r(  r)  r   r%  r   rS   r:  rc   rR   ra   rT   r_   r  )r   r!  r   Zhit_lfs_suffixr"  r   r   r   r5    s    

zUploadingCheck.is_lfsr   c                    sT    fdd|D }t dd |D }|jkr(td|d  djd  ddS )	af  
        Check a list of normal files to be uploaded.

        Args:
            file_path_list (List[Union[str, Path]]): The list of file paths to be checked.
            repo_type (str): The repo type, either `model` or `dataset`.

        Raises:
            ValueError: If the total size of normal files exceeds the limit.

        Returns: None
        c                    s   g | ]
} | s|qS r   )r5  r  rR  r   r   r   r   r!    r"  z5UploadingCheck.check_normal_files.<locals>.<listcomp>c                 S   s   g | ]}t |qS r   )r_   r  r   r   r   r!    rK  zTotal size of non-lfs files i   zMB and exceeds limit: MBN)sumr  r   )r   r   r   Znormal_file_listr  r   r  r   r3    s   

z!UploadingCheck.check_normal_files)r   N)r|  r}  r~  r  r,   r-   r.   r0   r/   rV  r   r2  r   r)  r   rd  rW  r	   rC  r5  r   r3  r   r   r   r   r     s,    

"&&r   )r2  rx  r   r0  r   r  r  r  r,  r   r  ra  r#  collectionsr   httpr   http.cookiejarr   Zos.pathr   pathlibr   typingr   r	   r
   r   r   r   r   r   r   r   r   r  r   Zrequests.adaptersr   r   Zrequests.exceptionsr   r  r   Zmodelscope.hub.constantsr   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   Zmodelscope.hub.errorsr6   r7   r8   r9   r:   r;   r<   r=   r>   r?   Zmodelscope.hub.gitr@   Zmodelscope.hub.inforA   rB   r  rD   Zmodelscope.hub.utils.aigcrE   Zmodelscope.hub.utils.utilsrF   rG   rH   rI   rJ   rK   rL   Zmodelscope.utils.constantrM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   Zmodelscope.utils.file_utilsr^   r_   Zmodelscope.utils.loggerr`   Zmodelscope.utils.repo_utilsra   rb   rc   rd   re   rf   rg   rh   Zmodelscope.utils.thread_utilsri   r   rj   ry   r   r   r   r   r   <module>   s   (0$L
(                     / 