o
    W+ i	                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZ d dl m!Z! d dl"Z"d dl#Z#d d	l#m$Z$ d d
l%m&Z&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZL d dlMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZW d dlXmYZY d dlZm[Z[m\Z\ d dl]m^Z^ d dl_m`Z` d dlambZbmcZcmdZdmeZemfZfmgZgmhZh d dlimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZz d dl{m|Z|m}Z} d dl~mZ d dlmZmZmZmZmZmZmZmZ d dlmZ e ZG dd dZG dd dZG dd dZdS )    N)defaultdict)
HTTPStatus)	CookieJar)
expanduserPath)AnyBinaryIODictIterableListOptionalTupleUnion)	urlencode)Session)HTTPAdapterRetry)	HTTPErrortqdm) API_HTTP_CLIENT_MAX_RETRIESAPI_HTTP_CLIENT_TIMEOUTAPI_RESPONSE_FIELD_DATAAPI_RESPONSE_FIELD_EMAIL#API_RESPONSE_FIELD_GIT_ACCESS_TOKENAPI_RESPONSE_FIELD_MESSAGEAPI_RESPONSE_FIELD_USERNAMEDEFAULT_MAX_WORKERSDEFAULT_MODELSCOPE_INTL_DOMAINMODELSCOPE_CLOUD_ENVIRONMENTMODELSCOPE_CLOUD_USERNAMEMODELSCOPE_CREDENTIALS_PATHMODELSCOPE_DOMAINMODELSCOPE_PREFER_AI_SITEMODELSCOPE_REQUEST_IDMODELSCOPE_URL_SCHEMEONE_YEAR_SECONDSREQUESTS_API_HTTP_METHODTEMPORARY_FOLDER_NAME"UPLOAD_BLOB_TQDM_DISABLE_THRESHOLDUPLOAD_COMMIT_BATCH_SIZEUPLOAD_MAX_FILE_COUNTUPLOAD_MAX_FILE_COUNT_IN_DIRUPLOAD_MAX_FILE_SIZE#UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT$UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFSVALID_SORT_KEYSDatasetVisibilityLicensesModelVisibility
VisibilityVisibilityMap)
InvalidParameterNotExistErrorNotLoginExceptionRequestErrordatahub_raise_on_errorhandle_http_post_errorhandle_http_responseis_okraise_for_http_statusraise_on_error)GitCommandWrapper)DatasetInfo	ModelInfo
Repository)	AigcModel)add_content_to_file
get_domainget_endpointget_readable_folder_sizeget_release_datetimeis_env_truemodel_id_to_group_owner_name)DEFAULT_DATASET_REVISIONDEFAULT_MODEL_REVISIONDEFAULT_REPOSITORY_REVISIONMASTER_MODEL_BRANCHMETA_FILES_FORMATREPO_TYPE_DATASETREPO_TYPE_MODELREPO_TYPE_SUPPORTConfigFieldsDatasetFormationsDatasetMetaFormatsDownloadChannelDownloadMode
Frameworks	ModelFileTasksVirgoDatasetConfig)get_file_hashget_file_size)
get_logger)DATASET_LFS_SUFFIXDEFAULT_IGNORE_PATTERNSMODEL_LFS_SUFFIXCommitHistoryResponse
CommitInfoCommitOperationCommitOperationAdd	RepoUtils)thread_executorc                   @   s.  e Zd ZdZdeefdee fddZdefddZ	dd
ee
 fddZ		ddee dee fddZejejdddddfdedee dee dee dee dee dee ded defddZ			ddededee dee ded defddZddedee fddZddedee fd d!Zedfded"ee dee defd#d$Zdd%d&ed'ee defd(d)Zedd*d&ed"ee dee defd+d,Zddd*d&ed"ee dee defd-d.Zeedd/d&ed'ee d"ee dee deeef f
d0d1Zddd	dd2d&ed'ee dee d3ee
 dee de
fd4d5Z dd&ed'edee fd6d7Z!e"d8d9 Z#ejejdd:de$dddf	ded;edee dee dee d<ee d=ee d"ee dee d>eee%e ef  d?eeee%e f  fd@dAZ&	B	C	ddDedEee dFee dee de'f
dGdHZ(dBdCddddIdDedEee dFee dJee dKee dee de'fdLdMZ)ddNee
e*f de*fdOdPZ+			ddedQee dNee
e*f de%e fdRdSZ,				ddedQee dNee
e*f dee de%e f
dTdUZ-dVdW Z.			ddedXee* dee fdYdZZ/			ddedXee* dee fd[d\Z0			ddedNee
e*f dee de1e%e e%e f fd]d^Z2		ddedNee
e*f de1e%e e%e f fd_d`Z3edd	d	i dfded"ee daee dbee
 dNee
e*f dcee' dee de%e' fdddeZ4ddfd&edged"ee fdhdiZ5deje6jddfdjedkedee dee dee dlee dee defdmdnZ7ddoedee fdpdqZ8ddjedkedee fdrdsZ9	t	B	u	ddjedked"edvedbe
dEedFedee fdwdxZ:ee$dBdyddzd&ed'ee d"ee dEedFedee fd{d|Z;e$d}dtdBdudd~d&ed"edvedbe
dEedFedee fddZ<e$dfdoed"ee dee fddZ=	ddjedkedoed"edee f
ddZ>e"dedefddZ?	ddjedked"ededede@dee fddZAe"deBjCfddZDeEd	dtdfdedjedked"ee dee
 dee
 dee fddZFeEdfdedjedked"ee dee f
ddZGeEdfdjedked"ee dee fddZHeEdfdjedkede
d"ee dee f
ddZIddoedede'fddZJ	ddjedked"ededee f
ddZK	ddee fddZL	ddedjedked"edee defddZM	ddedjedked"edee defddZNdd ZO	ddjedkede
dee ddf
ddZPdd ZQdd&edee defddZRdeSjedejdd	dtdd	d&edeee
df dee d'ee dee dee dee dee
 dee
 deeT defddZUddee$ddddd&edeVeW d<edee ded'ee d"ee dee dededeXfddZYdedddBdd	e$ddeeeZe[e\f ded&edeedf d'ee d<ee dee dee dee dee
 d"ee deXfddǄZ]ddddedde^e$dȜ	d&edeeeZe%e e%eZ f dee d<ee dee deedf d'ee deee%e ef  deee%e ef  ded"ee deeXe%eX f fdd΄Z_d	ddBdϜd&ed'edededeeeZe[e\f dee
 dee dee de'fddԄZ`dd՜d&ed'ede%eaeebf  dee de%eaeebf  f
dd؄Zc		ddeeeZe%e e%eZ f dedeee%e ef  deee%e ef  de%eede@f  f
ddۄZee"deVeW d<edeaeebf fdd݄ZfddegfddZhedd*d&ed'edeee%e f d"ee dee deaeebf fddZidS )HubApizModel hub api interface.
    Nendpointc                 C   s   |dur|nt  | _dt i| _t | _t|dddddd}t|d}| j	d	| | j	d
| t
D ]}t| j|tjt| j||d q4t | _dS )u   The ModelScope HubApi。

        Args:
            endpoint (str, optional): The modelscope server http|https address. Defaults to None.
        N
user-agent      )  i  i  i  F)totalreadconnectbackoff_factorstatus_forcelistrespect_retry_after_header)max_retrieszhttp://zhttps://timeout)rI   rl   ModelScopeConfigget_user_agentheadersr   sessionr   r   mountr(   setattr	functoolspartialgetattrUploadingCheckupload_checker)selfrl   ry   rw   retryadaptermethod r   ^/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/modelscope/hub/api.py__init__e   s.   	

zHubApi.__init__access_tokenc                 C   sL   ddl m} ddlm} | jr|| jjnt }| }|jd||dd |S )z
        Get jar cookies for authentication from access_token.

        Args:
            access_token (str): user access token on ModelScope.

        Returns:
            jar (CookieJar): cookies for authentication.
        r   )RequestsCookieJar)urlparsem_session_id/)domainpath)Zrequests.cookiesr   urllib.parser   rl   netlocrH   set)r   r   r   r   r   jarr   r   r   _get_cookies   s   
zHubApi._get_cookiesFcookies_requiredc                 C   s2   |r	| j |d}nt }|du r|rtd|S )a  
        Get cookies for authentication from local cache or access_token.

        Args:
            access_token (str): user access token on ModelScope
            cookies_required (bool): whether to raise error if no cookies found, defaults to `False`.

        Returns:
            cookies (CookieJar): cookies for authentication.

        Raises:
            ValueError: If no credentials found and cookies_required is True.
        )r   NzNo credentials found.You can pass the `--token` argument, or use HubApi().login(access_token=`your_sdk_token`). Your token is available at https://modelscope.cn/my/myaccesstoken)r   rz   get_cookies
ValueError)r   r   r   cookiesr   r   r   r      s   zHubApi.get_cookiesc                 C   s   |du r
t jd}|sdS |s| j}| d}| jj|d|i| | jd}t| |	 }t
| |t t }|j}t| t| t|t t |t t  |t t |fS )a|  Login with your SDK access token, which can be obtained from
           https://www.modelscope.cn user center.

        Args:
            access_token (str): user access token on modelscope, set this argument or set `MODELSCOPE_API_TOKEN`.
            If neither of the tokens exist, login will directly return.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            cookies: to authenticate yourself to ModelScope open-api
            git_token: token to access your git repository.

        Note:
            You only have to login once within 30 days.
        NZMODELSCOPE_API_TOKENNNz/api/v1/loginZAccessToken)jsonr|   )osenvirongetrl   r}   postbuilder_headersr|   r?   r   r@   r   r   r   rz   
save_tokensave_cookiessave_user_infor   r   )r   r   rl   r   rdtokenr   r   r   r   login   s8   





zHubApi.login model_id
visibilitylicensechinese_nameoriginal_model_idr   
aigc_modelrF   returnc	                 C   sd  |du rt d| j|dd}	|s| j}t|\}
}|
|||||tjddd}|durg| d}|j|	| | j	|d	 |
|j|j|j|j|j|jpN||j|j|j|j|j|jd
 |jrf|j|d< n| d}| | j	}tdd }|dd| rd|d< | jj|||	|d}t| | }t | | d| }|dur|!| || |S )a  Create model repo at ModelScope Hub.

        Args:
            model_id (str): The model id in format {owner}/{name}
            visibility (int, optional): visibility of the model(1-private, 5-public), default 5.
            license (str, optional): license of the model, default apache-2.0.
            chinese_name (str, optional): chinese name of the model.
            original_model_id (str, optional): the base model id which this model is trained from
            endpoint: the endpoint to use, default to None to use endpoint specified in the class
            token (str, optional): access token for authentication
            aigc_model (AigcModel, optional): AigcModel instance for AIGC model creation.
                If provided, will create an AIGC model with automatic file upload.
                Refer to modelscope.hub.utils.aigc.AigcModel for details.

        Returns:
            str: URL of the created model repository

        Raises:
            InvalidParameter: If model_id is invalid or required AIGC parameters are missing.
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nmodel_id is required!Tr   r   ZMODELSCOPE_TRAIN_IDr   )r   NameChineseNamer5   LicenseZOriginalModelIdZTrainIdz/api/v1/models/aigcr   r|   rl   )TagShowNameCoverImagesZAigcTypeZTagDescriptionZVisionFoundationZ	BaseModelWeightsNameWeightsSha256WeightsSizeZ	ModelPathTriggerWordsZModelSourceZOfficialTagsz/api/v1/models.r   en_USzX-Modelscope-Accept-Languager   r   r|   /models/)"r7   r   rl   rM   r   r   r   preupload_weightsr   r|   updatetagcover_imagesZ	aigc_typedescriptionZbase_model_typeZbase_model_idweight_filenameweight_sha256weight_sizeZ
model_pathtrigger_wordsZmodel_sourceZofficial_tagsr   splitrstripendswithr}   r   r?   r   r@   Zupload_to_repo)r   r   r   r   r   r   rl   r   r   r   owner_or_groupnamebodyr   r|   Zintl_endr   r   Zmodel_repo_urlr   r   r   create_model   sh   !


zHubApi.create_modeltag_namec              	   C   s  |du rt d|du rt d| dv rt d| d| j|dd}|s*| j}t|\}}|durV| d	}	|j|| | j|d
 |j||||j	|j
|j|jd}
n| d| d}	d}||d}
| jj|	|
|| | jd}t| | }t| | d| d| }|S )a]  Create a tag for a model at ModelScope Hub.

        Args:
            model_id (str): The model id in format {owner}/{name}
            tag_name (str): The tag name (e.g., "v1.0.0")
            endpoint: the endpoint to use, default to None to use endpoint specified in the class
            token (str, optional): access token for authentication
            aigc_model (AigcModel, optional): AigcModel instance for AIGC model tag creation.
                If provided, will create an AIGC model tag with automatic parameters.
                Refer to modelscope.hub.utils.aigc.AigcModel for details.

        Returns:
            str: URL of the created tag

        Raises:
            InvalidParameter: If model_id, tag_name, ref, or description is invalid.
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nr   ztag_name is required!)mainmasterz
tag_name "zp" is not allowed. Please use a different tag name (e.g., "v1.0", "v1.1", "latest"). Reserved names: main, masterTr   z/api/v1/models/aigc/repo/tagr   )r   r   r   r   r   r   r   r   /api/v1/models/z	/repo/tagr   )ZTagNameRefr   r   z/tags/)r7   lowerr   rl   rM   r   r   r|   r   r   r   r   r   r}   r   r?   r   r@   )r   r   r   rl   r   r   r   r   r   r   r   revisionr   r   Ztag_urlr   r   r   create_model_tagQ  sP   


zHubApi.create_model_tagc                 C   sb   t  }|s	| j}|du rtd| d| }| jj||| | jd}t| t	|
  dS )a0  Delete model_id from ModelScope.

        Args:
            model_id (str): The model id.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Raises:
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        N)Token does not exist, please login first.r   r   r|   rz   r   rl   r   r}   deleter   r|   r?   r@   r   )r   r   rl   r   r   r   r   r   r   delete_model  s   
zHubApi.delete_modelc                 C   s   |s| j }| d| dS )Nr   z.gitrl   )r   r   rl   r   r   r   get_model_url  s   zHubApi.get_model_urlr   c           	      C   s   t  }t|\}}|s| j}|r| d| d| d| }n
| d| d| }| jj||| | jd}t|t	|| |j
tjkrWt| rO| t S t| t t| dS )a  Get model information at ModelScope

        Args:
            model_id (str): The model id.
            revision (str optional): revision of model.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            The model detail information.

        Raises:
            NotExistError: If the model is not exist, will throw NotExistError

        Note:
            model_id = {owner}/{name}
        r   r   
?Revision=r   N)rz   r   rM   rl   r}   r   r   r|   r=   loggerstatus_coder   OKr>   r   r   r8   r   r?   )	r   r   r   rl   r   r   r   r   r   r   r   r   	get_model  s    

zHubApi.get_model)	repo_typerepo_idr   c                C   s  t jt}|dur5| dkr5t| }z| j|||dd W |S  ty4   t	d| d| d  w t
t }t|d}| j|||d	st| d}td| d
| d| d z| j|||dd W |S  ty~   t	d| d| d|   w |S )a  Get proper endpoint for read operation (such as download, list etc.)
        1. If user has set MODELSCOPE_DOMAIN, construct endpoint with user-specified domain.
           If the repo does not exist on that endpoint, throw 404 error, otherwise return the endpoint.
        2. If domain is not set, check existence of repo in cn-site and ai-site (intl version) respectively.
           Checking order is determined by MODELSCOPE_PREFER_AI_SITE.
             a. if MODELSCOPE_PREFER_AI_SITE is not set ,check cn-site first before ai-site (intl version)
             b. otherwise check ai-site before cn-site
           return the endpoint with which the given repo_id exists.
           if neither exists, throw 404 error
        Nr   T)r   r   rl   re_raiseRepo z does not exist on r   )Zcn_site)r   rl   z not exists on z#, will try on alternative endpoint )r   rl   r   z not exists on either z or )r   r   r   r#   stripr&   repo_exists	Exceptionr   errorrL   r$   rI   warning)r   r   r   srl   Zcheck_cn_firstZprefer_endpointZalternative_endpointr   r   r   get_endpoint_for_read  s:   


zHubApi.get_endpoint_for_read)r   rl   c          
      C   s^   t |\}}| j|||d}| j|t||d}| j||d|d}tdi ||||d}	|	S )aD  Get model information including commit history.

        Args:
            repo_id (str): The model id in the format of
                ``namespace/model_name``.
            revision (str, optional): Specific revision of the model.
                Defaults to ``DEFAULT_MODEL_REVISION``.
            endpoint (str, optional): Hub endpoint to use. When ``None``,
                use the endpoint specified when initializing :class:`HubApi`.

        Returns:
            ModelInfo: The model detailed information returned by
            ModelScope Hub with commit history.
        )r   r   rl   r   r   r   rl   T)r   r   	recursiverl   commitsauthorsiblingsNr   )rM   r   list_repo_commitsrT   get_model_filesrC   )
r   r   r   rl   r   _Z
model_datar   r   
model_infor   r   r   r     s   zHubApi.model_infoc          
      C   sb   t |\}}| j|||d}| j|t||d}| j||ptd|d}tdi ||||d}	|	S )a>  Get dataset information including commit history.

        Args:
            repo_id (str): The dataset id in the format of
                ``namespace/dataset_name``.
            revision (str, optional): Specific revision of the dataset.
                Defaults to ``None``.
            endpoint (str, optional): Hub endpoint to use. When ``None``,
                use the endpoint specified when initializing :class:`HubApi`.

        Returns:
            DatasetInfo: The dataset detailed information returned by
            ModelScope Hub with commit history.
        )
dataset_idr   rl   r   T)r   r   r   rl   r   Nr   )rM   get_datasetr   rS   get_dataset_filesrN   rB   )
r   r   r   rl   r   r   Zdataset_datar   r   dataset_infor   r   r   r   :  s   zHubApi.dataset_info)r   r   rl   c                C   sN   |du s|t kr| j|||dS |tkr| j|||dS td| dt d)aR  Get repository information for models or datasets.

        Args:
            repo_id (str): The repository id in the format of
                ``namespace/repo_name``.
            revision (str, optional): Specific revision of the repository.
                Currently only effective for model repositories. Defaults to
                ``DEFAULT_MODEL_REVISION``.
            repo_type (str, optional): Type of the repository. Supported
                values are ``"model"`` and ``"dataset"``. If not provided,
                ``"model"`` is assumed.
            endpoint (str, optional): Hub endpoint to use. When ``None``,
                use the endpoint specified when initializing :class:`HubApi`.

        Returns:
            Union[ModelInfo, DatasetInfo]: The repository detailed information
            returned by ModelScope Hub.
        N)r   r   rl   Arg repo_type z# not supported. Please choose from r   )rT   r   rS   r   r7   rU   )r   r   r   r   rl   r   r   r   	repo_infoZ  s   zHubApi.repo_info)r   rl   r   r   r   c                C   s  |du r| j }|dur| tvrtd| |du s"|ddkr(td| | j|dd}t|\}}|durJ| tkrJ| d| d| }	n
| d	| d| }	| jj	|	|| 
| jd
}
t|
t||d}|dkrodS |dkr{|ryt|
dS td| d td| )a  
        Checks if a repository exists on ModelScope

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`, *optional*):
                `None` or `"model"` if getting repository info from a model. Default is `None`.
                TODO: support studio
            endpoint(`str`):
                None or specific endpoint to use, when None, use the default endpoint
                set in HubApi class (self.endpoint)
            re_raise(`bool`):
                raise exception when error
            token (`str`, *optional*): access token to use for checking existence.
        Returns:
            True if the repository exists, False otherwise.
        NzNot support repo-type: %sr   ro   z5Invalid repo_id: %s, must be of format namespace/nameFr   /api/v1/datasets/r   r      Ti  z%Check repo_exists return status code r   zOFailed to check existence of repo: %s, make sure you have access authorization.)rl   r   rU   r   countr   rM   rS   r}   r   r   r|   r=   r   r   warn)r   r   r   rl   r   r   r   r   r   r   r   coder   r   r   r   }  s6   

zHubApi.repo_existsc                 C   s\   |s| j }|tkr| || n|tkr| || ntd| dtd| d dS )a  
        Delete a repository from ModelScope.

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`):
                The type of the repository. Supported types are `model` and `dataset`.
            endpoint(`str`):
                The endpoint to use. If not provided, the default endpoint is `https://www.modelscope.cn`
                Could be set to `https://ai.modelscope.ai` for international version.
        r   z not supported.r   z deleted successfully.N)rl   rS   delete_datasetrT   r   r   r   info)r   r   r   rl   r   r   r   delete_repo  s   zHubApi.delete_repoc                 C   s`   t j| tj}tjtjtj	t
ji}t|d}t|| W d    d S 1 s)w   Y  d S )Nw)r   r   joinr\   CONFIGURATIONrV   	frameworkr[   Ztorchtaskr]   otheropenr   dump)	model_dircfg_filecfgfiler   r   r   _create_default_config  s   "zHubApi._create_default_configzupload modelr  commit_messager   ignore_file_pattern
lfs_suffixc              
      s8  t jdtdd 	 |d u rtd|d u rtdtj|r%tj|r)tdtj|t	j
}tj|sItdt	j
 d| d	 t| t }|d u rUtd
t|}t|}|
d u rdg }
t|
trl|
g}
|d u st|d u rxtd| |std|  | j|||||	d tj|t}t }td| d| d td| d zzt||d}||}||vrtd|  ||| ||| t|}|D ]"  d dkrtj| }tj|rt| qt j!|dd q|D ]6  d dkr3t" fdd|
D rqtj| }tj#|r-t $|tj|  qt %|| q|sEt&j&' (d}d||f }|d ur`t|trS|gn|}|D ]}|)| qW|j*|||d |d urs|+|| td| d| d W n	 t,y    w W t j!|dd d S t j!|dd w ) NzThis function is deprecated and will be removed in future versions. Please use git command directly or use HubApi().upload_folder insteadrn   )
stacklevelzmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.zNo z file found in z, creating a default one.zMust login before upload!z5Visibility and License cannot be empty for new model.zCreating new model [%s])r   r   r   r   r   zPushing folder z
 as model r   zTotal folder size z;, this may take a while depending on actual pushing size...)r  Z
clone_fromzCreating new branch %sr   T)ignore_errorsc                    s   g | ]
}t | d uqS N)research).0patternfr   r   
<listcomp>Q      z%HubApi.push_model.<locals>.<listcomp>z%Y_%m_%d_%H_%M_%Sz$[automsg] push model %s to hub at %s)r  Zlocal_branchZremote_branchzSuccessfully push folder z to remote repo [z].)-warningsr  DeprecationWarningr7   r   r   existsisfiler  r\   r  r   r   rk   r  rz   r   r9   listdirrJ   
isinstancestrr   r  r   r)   rA   rE   Zget_remote_branchesZ
new_branchcheckoutremoveshutilrmtreeanyisdircopytreecopydatetimenowstrftimeZadd_lfs_typepushZtag_and_pushr   )r   r   r  r   r   r   r  r   r   r   r  r  r  r   Zfiles_to_saveZfolder_sizeZtmp_dirZgit_wrapperrepobranchesZfiles_in_reposrcdateZlfs_suffix_listsuffixr   r!  r   
push_model  s   /







"zHubApi.push_modelro   
   r   page_number	page_sizec           	      C   s   t  }|s	| j}| d}| jj|d|||f || | jd}t|t|| |j	t
jkrDt| r<| t }|S t| t t| dS )a  List models in owner or group.

        Args:
            owner_or_group(str): owner or group.
            page_number(int, optional): The page number, default: 1
            page_size(int, optional): The page size, default: 10
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Raises:
            RequestError: The request error.

        Returns:
            dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group}
        r   z.{"Path":"%s", "PageNumber":%s, "PageSize": %s})datar   r|   N)rz   r   rl   r}   putr   r|   r=   r   r   r   r   r>   r   r   r:   r   r?   )	r   r   r?  r@  rl   r   r   r   rA  r   r   r   list_modelsl  s(   

zHubApi.list_models)r?  r@  sortr  rl   rD  r  c                C   s   |s| j }| d}||d}|r%|tvr!td| dtt ||d< |r+||d< |r1||d< t }	| | j}
| jj	|||	|
d}t
| | }|	d	d
u r\d|v r\|d S |	dpbd}t|)a-  List datasets via OpenAPI with pagination, filtering and sorting.

        Args:
            owner_or_group (str): Search by dataset authors (including organizations and individuals).
            page_number (int, optional): The page number. Defaults to 1.
            page_size (int, optional): The page size. Defaults to 10.
            sort (str, optional): Sort key. If not provided, the server's default sorting is used.
                choose from ['default', 'downloads', 'likes', 'last_modified'].
            search (str, optional): Search by substring keywords in the dataset's Chinese name,
                English name, and authors (including organizations and individuals).
            endpoint (str, optional): Hub endpoint to use. When None, use the endpoint specified in the class.

        Returns:
            dict: The OpenAPI data payload, e.g.
                  {
                    "datasets": [...],
                    "total_count": int,
                    "page_number": int,
                    "page_size": int
                  }
        z/openapi/v1/datasets)r?  r@  zInvalid sort key: z. Supported sort keys: rD  r  r   paramsr   r|   successTrA  messagezFailed to list datasets)rl   r1   r7   listrz   r   r   r|   r}   r   r?   r   r:   )r   r   r?  r@  rD  r  rl   r   rF  r   r|   r   respmsgr   r   r   list_datasets  s<   
zHubApi.list_datasetsuse_cookiesc                 C   s@   d }t |tr|}|S t |trt }|r|d u rtd|S )Nr   )r*  r   boolrz   r   r   )r   rM  r   r   r   r   _check_cookie  s   

zHubApi._check_cookiecutoff_timestampc                 C   s.   | j |||d}|rdd |D }|S g }|S )a3  Get model branch and tags.

        Args:
            model_id (str): The model id
            cutoff_timestamp (int): Tags created before the cutoff will be included.
                                    The timestamp is represented by the seconds elapsed from the epoch time.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        )r   rP  rM  c                 S      g | ]}|d  qS Revisionr   r  xr   r   r   r#        z/HubApi.list_model_revisions.<locals>.<listcomp>)list_model_revisions_detail)r   r   rP  rM  Ztags_detailstagsr   r   r   list_model_revisions  s   zHubApi.list_model_revisionsc           
      C   s   |  |}|du rt }|s| j}| d| d| }| jj||| | jd}t|t|| |	 }t
| |t }	|	d d S )a  Get model branch and tags.

        Args:
            model_id (str): The model id
            cutoff_timestamp (int): Tags created before the cutoff will be included.
                                    The timestamp is represented by the seconds elapsed from the epoch time.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        Nr   z/revisions?EndTime=%sr   RevisionMapTags)rO  rK   rl   r}   r   r   r|   r=   r   r   r@   r   )
r   r   rP  rM  rl   r   r   r   r   r  r   r   r   rW    s   


z"HubApi.list_model_revisions_detailc                 C   s"   |D ]}|d |kr|  S qd S )NrS  r   )r   detailsr   itemr   r   r   get_branch_tag_detail  s
   zHubApi.get_branch_tag_detailr   c                    s6  |s| j }t  tttj  }| j||d u rdn||d\}}|r,dd |D ng }|r7dd |D ng }	 |t kru|d u rLt	}t
d|  ||vr\||	vr\td||f | ||}
|
d u rl| ||}
t
d|  |
S |d ur||v r| ||}
|
S t|d	kr|d u s|t	krt	}ntd
||f | ||}
|
S |d u r|r fdd|D ng }t|d	kr|d	 d }|d	 }
nt	}| ||}
dd|	 }t
d|  t
d|  |
S ||	vr|t	krt
d | ||}
ndd|	 }td|||f | ||}
t
d|  |
S )NF)rM  rl   c                 S   rQ  rR  r   rT  r   r   r   r#  1  rV  z4HubApi.get_valid_revision_detail.<locals>.<listcomp>c                 S   rQ  rR  r   rT  r   r   r   r#  2  rV  z9Model revision not specified, using default [%s] version.z$The model: %s has no revision : %s .z!Development mode use revision: %sr   z#The model: %s has no revision: %s !c                    s   g | ]
}|d   kr|qS )Z	CreatedAtr   rT  Zrelease_timestampr   r   r#  M  s    rS  z[%s],z5Model revision should be specified from revisions: %sz.Model revision not specified, use revision: %sz?Using the master branch is fragile, please use it with caution!z0The model: %s has no revision: %s valid are: %s!z%Use user-specified model revision: %s)rl   rK   introundr4  r5  	timestamp#get_model_branches_and_tags_detailsr'   rQ   r   r  r8   r^  debuglenr  r   )r   r   r   r   rl   Zcurrent_timestampZall_branches_detailZall_tags_detailZall_branchesZall_tagsZrevision_detailZrevisions_detailZvlr   r_  r   get_valid_revision_detail#  sx   
'


z HubApi.get_valid_revision_detailc                 C   s   | j ||||dd S )N)r   r   r   rl   rS  )rg  )r   r   r   r   rl   r   r   r   get_valid_revisionh  s   zHubApi.get_valid_revisionc           	      C   s|   |  |}|s
| j}| d| d}| jj||| | jd}t|t|| | }t	| |t
 }|d d |d d fS )a  Get model branch and tags.

        Args:
            model_id (str): The model id
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        r   z
/revisionsr   rZ  ZBranchesr[  )rO  rl   r}   r   r   r|   r=   r   r   r@   r   )	r   r   rM  rl   r   r   r   r   r  r   r   r   rd  r  s   


z*HubApi.get_model_branches_and_tags_detailsc                 C   sF   | j ||d\}}|rdd |D ng }|rdd |D ng }||fS )aq  Get model branch and tags.

        Args:
            model_id (str): The model id
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        )r   rM  c                 S   rQ  rR  r   rT  r   r   r   r#    rV  z6HubApi.get_model_branches_and_tags.<locals>.<listcomp>c                 S   rQ  rR  r   rT  r   r   r   r#    rV  )rd  )r   r   rM  Zbranches_detailZtags_detailr9  rX  r   r   r   get_model_branches_and_tags  s   
z"HubApi.get_model_branches_and_tagsrootr   r|   c                 C   s  |s| j }|rd||||f }nd|||f }| |}	|dur'|d|  }|du r.| jn|}tt j|d< | jj||	|d}
t	|
t
|	| |
 }t| g }|t d sgt
d| d	| d
 |S |t d D ]}|d dks{|d dkr|qm|| qm|S )a  List the models files.

        Args:
            model_id (str): The model id
            revision (Optional[str], optional): The branch or tag name.
            root (Optional[str], optional): The root path. Defaults to None.
            recursive (Optional[bool], optional): Is recursive list files. Defaults to False.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            headers: request headers
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            List[dict]: Model file list.
        z7%s/api/v1/models/%s/repo/files?Revision=%s&Recursive=%sz+%s/api/v1/models/%s/repo/files?Recursive=%sNz&Root=zX-Request-IDr   FileszNo files found in model z at revision r   r   z
.gitignorez.gitattributes)rl   rO  r|   r+  uuiduuid4hexr}   r   r=   r   r   r@   r   r   append)r   r   r   rj  r   rM  r|   rl   r   r   r   r   filesr  r   r   r   r     s:   
zHubApi.get_model_files)r   filenamec                C   s<   t  }| j|d||du rdn|d}dd |D }||v S )ap  Get if the specified file exists

        Args:
            repo_id (`str`): The repo id to use
            filename (`str`): The queried filename, if the file exists in a sub folder,
                please pass <sub-folder-name>/<file-name>
            revision (`Optional[str]`): The repo revision
        Returns:
            The query result in bool value
        TNF)r   r   rM  c                 S   rQ  r   r   )r  r  r   r   r   r#    rV  z&HubApi.file_exists.<locals>.<listcomp>)rz   r   r   )r   r   rq  r   r   rp  r   r   r   file_exists  s   zHubApi.file_existsdataset_name	namespacer   c                 C   s   |d u s|d u rt dt }|d u rtd|s| j}| d}	d |fd |fd |fd |fd |fd |fd}
| jj|	|
|| | jd}t	||	|
 t
|  | d| d| }td|  |S )	Nz(dataset_name and namespace are required!r   z/api/v1/datasets)r   r   ZOwnerr   r5   Description)rp  r   r|   z
/datasets/r   zCreate dataset success: )r7   rz   r   r   rl   r}   r   r   r|   r<   r@   r   r   r  )r   rs  rt  r   r   r   r   rl   r   r   rp  r   Zdataset_repo_urlr   r   r   create_dataset  s4   	
	
zHubApi.create_datasetr   c                 C   sb   t  }|s	| j}|d u rtd| d| }| jj||| | jd}t| t	|
  d S )Nr   r  r   r   )r   r   rl   r   r   r   r   r   r   r    s   
zHubApi.delete_datasetc           
      C   sj   |s| j }| d| d| }t }| jj||d}| }t||| |d d }|d d }	||	fS )z Get the dataset id and type. r  r   r   DataZIdType)rl   rz   r   r}   r   r   r;   )
r   rs  rt  rl   datahub_urlr   r   rJ  r   dataset_typer   r   r   get_dataset_id_and_type.  s   zHubApi.get_dataset_id_and_typeTd   	root_pathc	                 C   s   t dt | j|||d\}	}
|rdnd}|s| j}| d|	 d}|r'|nd|r,|nd|||d	}t }| jj|||d
}|	 }t
||| |S )z?
        @deprecated: Use `get_dataset_files` instead.
        zMThe function `list_repo_tree` is deprecated, use `get_dataset_files` instead.rs  rt  rl   TrueFalser  
/repo/treer   r   rS  ZRootZ	Recursive
PageNumberPageSizerF  r   )r%  r  r&  r|  rl   rz   r   r}   r   r   r;   )r   rs  rt  r   r~  r   r?  r@  rl   dataset_hub_idr{  rz  rF  r   r   rJ  r   r   r   list_repo_tree;  s&   

zHubApi.list_repo_tree2   )r   r   r?  r@  rl   c             
   C   s  ddl m} ||r|ddkr|d\}}	ntd| d|s&| j}|r3| d| d| d	n| d
| d	}
|p>t||d}t }z+| j	j
|
||| | jd}t| | }t| |
dtjkrpt|W S W dS  tjjy } ztd| dt| d}~ww )a@  
        Get the commit history for a repository.

        Args:
            repo_id (str): The repository id, in the format of `namespace/repo_name`.
            repo_type (Optional[str]): The type of the repository. Supported types are `model` and `dataset`.
            revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
            page_number (int): The page number for pagination. Defaults to 1.
            page_size (int): The number of commits per page. Defaults to 50.
            endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

        Returns:
            CommitHistoryResponse: The commit history response.

        Examples:
            >>> from modelscope.hub.api import HubApi
            >>> api = HubApi()
            >>> commit_history = api.list_repo_commits('meituan/Meeseeks')
            >>> print(f"Total commits: {commit_history.total_count}")
            >>> for commit in commit_history.commits:
            ...     print(f"{commit.short_id}: {commit.title}")
        r   is_relative_pathr   ro   Invalid repo_id:  !z/api/v1/s/z/commitsr   )r   r  r  rE  Codez%Failed to get repository commits for : N)datasets.utils.file_utilsr  r  r   r   rl   rP   rz   r   r}   r   r   r|   r?   r   r@   r   r   re   Zfrom_api_responserequests
exceptionsRequestExceptionr   r+  )r   r   r   r   r?  r@  rl   r  _owner_dataset_nameZcommits_urlrF  r   r   rJ  er   r   r   r   \  s8   
zHubApi.list_repo_commitsr   )r   r~  r   r?  r@  rl   c                C   s   ddl m} ||r|ddkr|d\}	}
ntd| d| j|
|	|d\}}|s0| j}| d| d	}|||r>d
nd||d}t }| j	j
|||d}| }t||| |d d S )aZ  
        Get the dataset files.

        Args:
            repo_id (str): The repository id, in the format of `namespace/dataset_name`.
            revision (str): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
            root_path (str): The root path to list. Defaults to '/'.
            recursive (bool): Whether to list recursively. Defaults to True.
            page_number (int): The page number for pagination. Defaults to 1.
            page_size (int): The number of items per page. Defaults to 100.
            endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

        Returns:
            List: The response containing the dataset repository tree information.
                e.g. [{'CommitId': None, 'CommitMessage': '...', 'Size': 0, 'Type': 'tree'}, ...]
        r   r  r   ro   r  r  r  r  r  r  r  r  r  rx  rk  )r  r  r  r   r   r|  rl   rz   r   r}   r   r   r;   )r   r   r   r~  r   r?  r@  rl   r  r  r  r  r{  rz  rF  r   r   rJ  r   r   r   r     s*   

zHubApi.get_dataset_filesc                 C   sx   t  }|s	| j}|r| d| d| }n| d| }| jj||| | jd}t| | }t	||| |t
 S )ac  
        Get the dataset information.

        Args:
            dataset_id (str): The dataset id.
            revision (Optional[str]): The revision of the dataset.
            endpoint (Optional[str]): The endpoint to use, defaults to None to use the endpoint specified in the class.

        Returns:
            dict: The dataset information.
        r  r   r   )rz   r   rl   r}   r   r   r|   r?   r   r;   r   )r   r   r   rl   r   r   r   rJ  r   r   r   r     s   zHubApi.get_datasetc                 C   s   |s| j }| d| d| }t }| jj||| | jd}| }	t||	| |	d }
|
du r@t	d| d| d| d	|
d
 }
|
S )z( Get the meta file-list of the dataset. r  z/repo/tree?Revision=r   rx  Nz'The modelscope dataset [dataset_name = z, namespace = z, version = z] dose not existrk  )
rl   rz   r   r}   r   r   r|   r   r;   r8   )r   rs  rt  r   r   rl   rz  r   r   rJ  	file_listr   r   r   get_dataset_meta_file_list  s&   
z!HubApi.get_dataset_meta_file_listr{  meta_cache_dirc                 C   sX   t j|t|  tjj }t|d}|d W d   dS 1 s%w   Y  dS )z
        Dump the data_type as a local file, in order to get the dataset
         formation without calling the datahub.
        More details, please refer to the class
        `modelscope.utils.constant.DatasetFormations`.
        r
  z3*** Automatically-generated file, do not modify ***N)	r   r   r  r+  rW   Zformation_mark_extvaluer  write)r{  r  Zdataset_type_file_pathfpr   r   r   dump_datatype_file  s   "zHubApi.dump_datatype_filer  c              
   C   s(  t t}t|}	t|	 }
t }tj||d |s| j}|D ]q}|d }t	j
|d }||
v r| d| d| d| d| 	}| jj||d}t| t	j
||}t	j
|rmtd	| d
| d || | qt|d}||j W d    n1 sw   Y  || | q||	fS )N)r{  r  r   r   r  r   /repo?Revision=
&FilePath=rw  zReusing dataset z's python file ()wb)r   rI  rW   rX   rz   r   rk   r  rl   r   r   splitextr}   r   r?   r  r'  r   r   ro  r  r  content)r   rs  rt  r   r  r{  r  rl   Zlocal_pathsZdataset_formationZdataset_meta_formatr   	file_info	file_path	extensionrz  r   Z
local_pathr"  r   r   r   "get_dataset_meta_files_local_paths  s>   z)HubApi.get_dataset_meta_files_local_paths   c                    s  ddl }ddlm} ddl}tj||| jdd	 }|t
jkr.tj|r.t| tj|r>td|  |S t }td tj| |dd	}t|jd
d}	||	dd}
 fdd}d}t|dQ}||D ]D}|
t| | drdd |D }t|dkrqp|dkrd}nd}||}|j|d|dd |d7 }qp|D ]	}||d  qqpW d   n1 sw   Y  |
  |S )zO
        Fetch the meta-data files from the url, e.g. csv/jsonl files.
        r   Nr   zUTF-8encodingzReusing cached meta-data file: zLoading meta-data file ...T)r   streamzcontent-length)rq   Zdynamic_ncolsc                 3   sH    g }|   D ]}|d}|| t| kr|V  g }q|V  d S )Nutf-8)
iter_linesdecodero  rf  )rJ  Z
chunk_datarA  
chunk_sizer   r   	get_chunkL  s   


z3HubApi.fetch_meta_files_from_url.<locals>.get_chunkaZjsonlc                 S   s   g | ]}|  rt|qS r   )r   r   loads)r  liner   r   r   r#  [  s    z4HubApi.fetch_meta_files_from_url.<locals>.<listcomp>F\)indexheader
escapecharro   
)hashlib	tqdm.autor   Zpandasr   r   r  md5encode	hexdigestrZ   ZFORCE_REDOWNLOADr'  r-  r   r  rz   r   r  r   ra  r|   r  r   rf  r   Z	DataFrameZto_csvr  close)urlZout_pathr  moder  r   pdr   response
total_sizeprogressr  Ziter_numr"  chunkZwith_headerZchunk_dfr  r   r  r   fetch_meta_files_from_url5  sH    





z HubApi.fetch_meta_files_from_url	file_nameviewextension_filterc           
      C   sR   |r|r|s
t dd|||d}t|}|s| j}| d| d| d| }	|	S )Nz:Args (file_name, dataset_name, namespace) cannot be empty!ZSDK)ZSourcerS  FilePathZViewr  r   /repo?)r   r   rl   )
r   r  rs  rt  r   r  r  rl   rF  file_urlr   r   r   get_dataset_file_urlm  s   
zHubApi.get_dataset_file_urlc              	   C   sF   |s| j }|r!tj|d tv r!| d| d| d| d| 	}|S )Nr   r  r   r  r  )rl   r   r   r  rR   )r   r  rs  rt  r   rl   r   r   r   get_dataset_file_url_origin  s   z"HubApi.get_dataset_file_url_originc                 C   s.   |s| j }| d| d| d| }| |S )Nr  r   /ststoken?Revision=)rl   datahub_remote_call)r   rs  rt  r   rl   rz  r   r   r   get_dataset_access_config  s   
z HubApi.get_dataset_access_configcheck_cookiec           
      C   sp   |s| j }| d| d| d| }|r| jdd}nt }| jj||| | jd}| }	t	|	 |	d S )Nr  r   r  T)rM  r  r   r|   rx  )
rl   rO  rz   r   r}   r   r   r|   r   r@   )
r   rs  rt  r  r   rl   rz  r   r   rJ  r   r   r   !get_dataset_access_config_session  s    
z(HubApi.get_dataset_access_config_sessionversionc           
      C   s   t jtjd}|stdtj | d}tjt	
 }t||d}t|d}| jj|||| | jdd}| }	|	d d	krJtd
|	 |	d S )z.
        Get virgo dataset meta info.
        r   z"Virgo endpoint is not set in env: z/data/set/download)Z	dataSetIdZdataSetVersion)rA  i  )r  r   r   r|   ry   r  r   zFailed to get virgo dataset: rA  )r   r   r   r^   Zenv_virgo_endpointRuntimeErrorr  utilsdict_from_cookiejarrz   r   dictr}   r   r   r|   r   )
r   r   r  Zvirgo_endpointZvirgo_dataset_urlr   r   rA  r   rJ  r   r   r   get_virgo_meta  s,   

zHubApi.get_virgo_metazip_file_namec                 C   s   |s| j }| d| d| }t }| jj||| | jd}| }	t|	 |	d }
t	|
d }| d| }| jj||| | jd}| }t| |d }|d d | d | d | }||d	< |S )
Nr  r   r  rx  r5   r  z	-unzippedr   ZDir)
rl   rz   r   r}   r   r   r|   r   r@   r6   )r   rs  rt  r   r  rl   rz  r   r   rJ  rA  r   Zdatahub_sts_urlZr_stsZresp_stsZdata_stsZfile_dirr   r   r   &get_dataset_access_config_for_unzipped  s*   



 z-HubApi.get_dataset_access_config_for_unzippedc                 C   sl   |s| j }| d| d| d| d| d| d| }t }	| jj||	dd}
|
 }
t|
 |
d	 }
|
S )
Nr  r   z/oss/tree/?MaxLimit=
&Revision=z&Recursive=z&FilterDir=i  )r  r   ry   rx  )rl   rz   r   r}   r   r   r@   )r   rs  rt  	max_limitZis_recursiveZis_filter_dirr   rl   r  r   rJ  r   r   r   list_oss_dataset_objects  s"   zHubApi.list_oss_dataset_objectsobject_namec           	   	   C   v   |r|r|r|st d|s| j}| d| d| d| d| 	}t }| jj||d}| }t| |d }|S )NArgs cannot be empty!r  r   z
/oss?Path=r  r  r   Messager   rl   rz   r   r}   r   r   r@   	r   r  rs  rt  r   rl   r  r   rJ  r   r   r   delete_oss_dataset_object  s    z HubApi.delete_oss_dataset_objectc           	   	   C   r  )Nr  r  r   z/oss/prefix?Prefix=z/&Revision=r  r  r  r  r   r   r   delete_oss_dataset_dir  s   zHubApi.delete_oss_dataset_dirc                 C   s>   t  }| jj||dt  id}| }t||| |d S )Nrm   r   rx  )rz   r   r}   r   r{   r   r;   )r   r  r   r   rJ  r   r   r   r  &  s   
zHubApi.datahub_remote_calluse_streamingc              
   C   s.  t ddk}|s| j}|r|r|s|szbt }| d| d| d}| jj||| | jd}t	| t
jj}	d}
tt jv rEt jt }	tt jv rOt jt }
| d| d| d|	 d	|
 	}| jj||| | jd}| }t| W d S  ty } zt| W Y d }~d S d }~ww d S d S d S d S )
NZCI_TESTr  r  r   z/download/increaser   r   z/download/uv/z?user=)r   getenvrl   rz   r   r}   r   r   r|   r?   rY   ZLOCALr  r    r   r!   r   r@   r   r   r   )r   rs  rt  r  rl   Z
is_ci_testr   Zdownload_count_urlZdownload_count_respZchannel	user_nameZdownload_uv_urlZdownload_uv_respr  r   r   r   dataset_download_statistics0  s@   







z"HubApi.dataset_download_statisticsc                 C   s   t tt ji|S r  )r%   r+  rl  rm  rn  )r   r|   r   r   r   r   Q  s   zHubApi.builder_headersc                 C   s.   | d\}}|s| j}| d| d| dS )Nr   r  r  )r   rl   )r   r   rl   Z
_namespacer  r   r   r   get_file_base_pathU  s   zHubApi.get_file_base_path)	r   r   r   r   r   rl   exist_okcreate_default_configr   r  r  c       	         K   s  |st d|s| j}| j||d | j||||d}|r>|r6| d| d| }td| d|  |S t d| d|d}t|d	krMt d
|\}}|tkrdd t	j
 D }|| }|du rrt d| d| j|||||
d}|	rt 4}ddlm} |||}dddd}|d}|si }i ||}t|dt|gdd W d   n1 sw   Y  td| ddd |S |tkrdd tj
 D }|| }|du rt d| d| j|||||d}td| ddd |S t d | d!t )"a%  
        Create a repository on the ModelScope Hub.

        Args:
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            token (Union[str, bool, None]): The access token.
            visibility (Optional[str]): The visibility of the repo,
                could be `public`, `private`, `internal`, default to `public`.
            repo_type (Optional[str]): The repo type, default to `model`.
            chinese_name (Optional[str]): The Chinese name of the repo.
            license (Optional[str]): The license of the repo, default to `apache-2.0`.
            endpoint (Optional[str]): The endpoint to use.
                In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
            exist_ok (Optional[bool]): If the repo exists, whether to return the repo url directly.
            create_default_config (Optional[bool]): If True, create a default configuration file in the model repo.
            **kwargs: The additional arguments.

        Returns:
            str: The repo url.
        Repo id cannot be empty!)r   rl   )r   rl   r   r   r  r   z already exists, got repo url: z already exists!rn   zBInvalid repo id, should be in the format of `owner_name/repo_name`c                 S       i | ]\}}| d s||qS __
startswithr  kvr   r   r   
<dictcomp>       z&HubApi.create_repo.<locals>.<dictcomp>NzInvalid visibility: z9, supported visibilities: `public`, `private`, `internal`)r   r   r   r   r   r   rD   Zpytorchztext-generationT)r  r  Zallow_remoteconfig_jsonzconfiguration.json)Zignore_push_errorz"New model created successfully at r   flushc                 S   r  r  r  r  r   r   r   r    r  )rs  rt  r   r   r   z$New dataset created successfully at Invalid repo type: , supported repos: )r   rl   r   r   r   r   r   rf  rT   r4   __dict__itemsr   upperr   tempfileTemporaryDirectorymodelscope.hub.repositoryrE   rG   r   dumpsprintrS   r2   rv  rU   )r   r   r   r   r   r   r   rl   r  r  r   kwargsr   repo_urlZrepo_id_listrt  	repo_nameZvisibilitiesZtemp_cache_dirrE   r8  Zdefault_configr  configr   r   r   create_repo\  s~   $




zHubApi.create_repo      )commit_descriptionr   r   r   rl   rw   ry   
operationsr  rw   ry   c                C   s|  |st d|s| j}|tvrt d| dt | d| d| d| }|p,d| }|p0d}| j|d	d
}| j||d}d}t|	D ]}z|dkrZtd|d  d| d tj	|| 
| jt|||
d}|jdkrz| }W n tjy   |j}Y nw d|j d| d| }d|j  krdk rn ntd|d  d|  n(t d| | }|di dd}td|  t||||dW   S W nP tjjy } z|}td|d  dt|  W Y d}~n0d}~w ty$ } z|}td |d  dt|  ||	d kr W Y d}~nd}~ww ||	d k r1td qEtjd!|	 d"| )#a  
        Create a commit on the ModelScope Hub with retry mechanism.

        Args:
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            operations (Iterable[CommitOperation]): The commit operations.
            commit_message (str): The commit message.
            commit_description (Optional[str]): The commit description.
            token (str): The access token. If None, will use the cookies from the local cache.
                See `https://modelscope.cn/my/myaccesstoken` to get your token.
            repo_type (Optional[str]): The repo type, should be `model` or `dataset`. Defaults to `model`.
            revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.
            endpoint (Optional[str]): The endpoint to use.
                In the format of `https://www.modelscope.cn` or 'https://www.modelscope.ai'
            max_retries (int): Number of max retry attempts (default: 3).
            timeout (int): Timeout for each request in seconds (default: 180).

        Returns:
            CommitInfo: The commit info.

        Raises:
            requests.exceptions.RequestException: If all retry attempts fail.
        r  r   r  /api/v1/repos/r  z/commit/z
Commit to r   Tr   )r  r  Nr   zAttempt ro   z to create commit for z...)r|   rA  r   ry   r  zHTTP z error from r  rp   iX  zServer error on attempt zClient request failed: rx  oidzCommit succeeded: )Z
commit_urlr  r  r  zRequest failed on attempt zUnexpected error on attempt zFailed to create commit after z attempts. Last error: )r   rl   rU   r   _prepare_commit_payloadranger   r  r  r   r   r|   r   r  r   JSONDecodeErrortextr   r   rf   r  r  r+  r   r   timesleep)r   r   r  r  r  r   r   r   rl   rw   ry   r  r   payloadZlast_exceptionattemptr  Zerror_detail	error_msgrJ  r  r  r   r   r   create_commit  s   %


*
zHubApi.create_commitz[Uploading])r   r   r  r  buffer_size_mb	tqdm_descdisable_tqdmr   path_or_fileobjpath_in_repor  r   r!  c             	   C   s  |t vrtd| dt  |std| j|dd t|ttfr5tjtj	|}|p3tj
|}n|s;tdt|tjrE| }| j| | jj|g|d |durZ|nd	| d
}|dkrhtdt||d}|d }|d }| j|||| jddd | j||||||
|	d}t|||d}| j||rdnd|_|d |_|g}td| ddd | j|||||||d}|S )a  
        Upload a file to the ModelScope Hub.

        Args:
            path_or_fileobj (Union[str, Path, bytes, BinaryIO]):
                The local file path or file-like object (BinaryIO) or bytes to upload.
            path_in_repo (str): The path in the repo to upload to.
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
                See `https://modelscope.cn/my/myaccesstoken` to get your token.
            repo_type (Optional[str]): The repo type, default to `model`.
            commit_message (Optional[str]): The commit message.
            commit_description (Optional[str]): The commit description.
            buffer_size_mb (Optional[int]): The buffer size in MB for reading the file. Default to 1MB.
            tqdm_desc (Optional[str]): The description for the tqdm progress bar. Default to '[Uploading]'.
            disable_tqdm (Optional[bool]): Whether to disable the tqdm progress bar. Default to False.
            revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

        Returns:
            CommitInfo: The commit info.

        Examples:
            >>> from modelscope.hub.api import HubApi
            >>> api = HubApi()
            >>> commit_info = api.upload_file(
            ...     path_or_fileobj='/path/to/your/file.txt',
            ...     path_in_repo='optional/path/in/repo/file.txt',
            ...     repo_id='your-namespace/your-repo-name',
            ...     commit_message='Upload file.txt to ModelScope hub'
            ... )
            >>> print(commit_info)
        r   r  z$Path or file object cannot be empty!Tr   z#Arg `path_in_repo` cannot be empty!file_path_listr   NzUpload z to ModelScope hubr   z4Buffer size: `buffer_size_mb` must be greater than 0)file_path_or_objr  	file_size	file_hashFr   r   r   rl   r  r  r   r   sha256sizerA  r!  r   r#  r"  file_hash_infolfsnormalis_uploadedzCommitting file to z ...r  r   r  r  r  r   r   r   )rU   r   r   r*  r+  r   r   r   abspathr   basenameioBufferedIOBaserr   r   
check_filecheck_normal_filesr_   r  rl   _upload_blobrh   is_lfs_upload_mode_is_uploadedr	  r  )r   r"  r#  r   r   r   r  r  r  r   r!  r   hash_info_dr'  r(  
upload_resZadd_operationr  commit_infor   r   r   upload_fileA  s|   0

zHubApi.upload_file)	r#  r  r  r   r   allow_patternsignore_patternsmax_workersr   folder_pathrA  rB  rC  c             
      s   st d|du rt dtvrt d dt j|dd |r'|nd}|	r-|	nd}	|	du r6g }	nt|	tr>|	g}	|	t7 }	|durH|nd  d	}|pQd
}td j||||	d}t	|dkrnt d| dtdt	| d j
jdd |D d j |jddd t|
dd fdd}|| |||ddd}g }|D ]-}|d }|d }|d }|d }t|||d }j
|rd!nd"|_||_|| qt	|dkrt d| dtdkrtnt	|}t	|d | d }td#t	| d$| d%| d&dd' g }tt|d(|d)D ]-}||| |d |  }| d*|d  d+| d,}j |||||d-}|| qt	|dkrV|d S |S ).a  
        Upload a folder to the ModelScope Hub.

        Args:
            repo_id (str): The repo id in the format of `owner_name/repo_name`.
            folder_path (Union[str, Path, List[str], List[Path]]): The folder path or list of file paths to upload.
            path_in_repo (Optional[str]): The path in the repo to upload to.
            commit_message (Optional[str]): The commit message.
            commit_description (Optional[str]): The commit description.
            token (Union[str, None]): The access token. If None, will use the cookies from the local cache.
                See `https://modelscope.cn/my/myaccesstoken` to get your token.
            repo_type (Optional[str]): The repo type, default to `model`.
            allow_patterns (Optional[Union[List[str], str]]): The patterns to allow.
            ignore_patterns (Optional[Union[List[str], str]]): The patterns to ignore.
            max_workers (int): The maximum number of workers to use for uploading files concurrently.
                Defaults to `DEFAULT_MAX_WORKERS`.
            revision (Optional[str]): The branch or tag name. Defaults to `DEFAULT_REPOSITORY_REVISION`.

        Returns:
            Union[CommitInfo, List[CommitInfo]]:
                The commit info or list of commit infos if multiple batches are committed.

        Examples:
            >>> from modelscope.hub.api import HubApi
            >>> api = HubApi()
            >>> commit_info = api.upload_folder(
            ...     repo_id='your-namespace/your-repo-name',
            ...     folder_path='/path/to/your/folder',
            ...     path_in_repo='optional/path/in/repo',
            ...     commit_message='Upload my folder',
            ...     token='your-access-token'
            ... )
            >>> print(commit_info.commit_url)
        z"The arg `repo_id` cannot be empty!Nz%The arg `folder_path` cannot be None!r   r  Tr   z
Upload to z on ModelScope hubzUploading fileszPreparing files to upload ...)folder_path_or_filesr#  rA  rB  r   z"No files to upload in the folder: r  z	Checking z files to upload ...c                 S   s   g | ]\}}|qS r   r   )r  r   r]  r   r   r   r#  	  rV  z(HubApi.upload_folder.<locals>.<listcomp>r$  Fr)  )rC  r!  c              	      sX   | \}}t |d}|d }|d }j ||||tkd| d d}|||d |dS )	N)r&  r'  r(  z[Uploading ]r*  r1  )file_path_in_repor  r1  r.  )r_   r9  r*   )Z	item_pairr
  rG  r  r=  r'  r(  r>  r   r   r   r   r   _upload_items'	  s(   
z+HubApi.upload_folder.<locals>._upload_itemsro   )r   r   r   r  r  r  r!  rG  r  r1  r.  r-  r/  r0  zCommitting z
 files in z batch(es) of size r   r  z[Committing batches] )descrq   z (batch r   r  r2  )r   rU   r   r*  r+  rc   r   r  _prepare_upload_folderrf  r   r8  r  rl   rj   rh   r:  r;  r<  ro  r+   r	  r   r  r  )r   r   rD  r#  r  r  r   r   rA  rB  rC  r   prepared_repo_objectsrI  Zuploaded_items_listr  Zitem_dZprepared_path_in_repoZprepared_file_pathr1  r.  optZcommit_batch_sizeZnum_batchesZcommit_infosiZbatch_operationsZbatch_commit_messager?  r   rH  r   upload_folder  s   1


	zHubApi.upload_folder)r!  r   r  r+  r,  rA  c             	   C   s  t d dd d d}	||dg}
| j|||
d}t|dkr |d nd }|d u r9td|d d  d	 d
|	d< |	S t }|rCt |nd }|d u rMtd| j	dd|d  i | 
| j}|d d fdd}t|dd
||d`}t|ttfrt|d}tj|d ||||d}W d    n1 sw   Y  n.t|trtj|d ||t||d}nt|tjrtj|d ||||d}ntdW d    n1 sw   Y  t|d | }t|d |d |	d< |d |	d< |d |	d< |	S )NF)r  r1  r   
status_msg)r  r,  )r   r   objectsro   r   zBlob    z  has already uploaded, reuse it.Tr1  r   Cookiezm_session_id=r   r  c                 s   s,    	 |  |}|sdS |t| |V  q)z8Lazy function (generator) to read a file piece by piece.TN)rr   r   rf  )Zfile_objectpbarr  Zckr   r   r   read_in_chunks	  s   
z+HubApi._upload_blob.<locals>.read_in_chunksB)rq   unitZ
unit_scalerJ  disablerbr  )r|   rA  zInvalid data type to uploadZrspr  r   r  rP  )r  _validate_blobrf  r   re  rz   r   r   r|   r   r   r   r*  r+  r   r  r  rB  bytesr5  BytesIOr6  r?   r   r@   )r   r   r   r+  r,  rA  r!  r   r  Zres_drQ  upload_objectsZupload_objectr   r|   rU  rT  r"  r  rJ  r   r   r   r9  z	  s~   	

!
zHubApi._upload_blobr   rQ  c                C   s   |s| j }| d| d| d}d|d}t }|du r!tdtj|| | jt	||d}t
|d	 | }	t|	d	 g }
|	d
 d }|D ]}|
|d d d |d d qI|
S )aC  
        Check the blob has already uploaded.
        True -- uploaded; False -- not uploaded.

        Args:
            repo_id (str): The repo id ModelScope.
            repo_type (str): The repo type. `dataset`, `model`, etc.
            objects (List[Dict[str, Any]]): The objects to check.
                oid (str): The sha256 hash value.
                size (int): The size of the blob.
            endpoint: the endpoint to use, default to None to use endpoint specified in the class

        Returns:
            List[Dict[str, Any]]: The result of the check.
        r  r  z/info/lfs/objects/batchZupload)	operationrQ  Nr   )r|   rA  r   rZ  rx  rQ  actionshrefr  )r  r  )rl   rz   r   r   r  r   r   r|   r   r  r?   r@   ro  )r   r   r   rQ  rl   r  r  r   r  rJ  r^  Zresp_objectsobjr   r   r   r[  	  s6   


zHubApi._validate_blobrE  c           	         s6  d  d }t |trtj|d r|}ntdtj|r"|g}n| |d u rQ| j  t 	 
    sBtd  d fddt dD ni |D ]}tj|rk| j| |tj|< qUttj ||d}|r|d	 d	nd
fdd|D }tdt| d |S )Nr   z0Uploading multiple folders is not supported now.zProvided path: 'z' is not a directoryc                    s$   i | ]}|  r|  |qS r   )is_filerelative_toas_posix)r  r   )rD  r   r   r  /
  s    z1HubApi._prepare_upload_folder.<locals>.<dictcomp>z**/*)rA  rB  r   r   c                    s    g | ]} | t | fqS r   )r+  )r  relpath)prefixrelpath_to_abspathr   r   r#  D
  s    z1HubApi._prepare_upload_folder.<locals>.<listcomp>z	Prepared z files for upload.)r*  rI  r   r   r(  r   r   check_folderr   r   resolveis_dirsortedglobr7  r4  ri   Zfilter_repo_objectskeysr   r   r  rf  )	r   rE  r#  rA  rB  Z
files_pathr   Zfiltered_repo_objectsrL  r   )rD  rg  rh  r   rK  
  sD   


zHubApi._prepare_upload_folderc              	   C   s  |g d}d}| D ]t}t |tr"|jr"td|j d |d7 }q	t |trJ|jdkrJ|jr1dnd|jd|jj	d	|
  d
d}|d | q	t |trp|jdkrp|jrYdnd|jd|jj	|jjd	d	d}|d | q	td| dt|dd |dkrtd| d |S )zN
        Prepare the commit payload to be sent to the ModelScope hub.
        )r  r`  r   zSkipping file 'z(' in commit (ignored by gitignore file).ro   r0  r   creater   base64)actionr   typer,  r+  r  r  r`  r/  z(Unknown operation to commit. Operation: z. Upload mode: r;  NzSkipped z/ file(s) in commit (ignored by gitignore file).)r*  rh   Z_should_ignorer   re  r#  r;  r<  Zupload_infor,  Z
b64contentr  ro  r+  r   r   r  )r  r  r  Znb_ignored_filesr_  Zcommit_actionr   r   r   r  M
  sJ   

		
zHubApi._prepare_commit_payload皙?internal_timeoutc                 C   s~   dt dtfdd}| j d}|||d}d}|dur=| }d	|v r*|d	 d
 }nd}|r=|||d}|dur=|j }|S )a  
        Get the internal acceleration domain.

        Args:
            internal_timeout (float): The timeout for the request. Default to 0.2s

        Returns:
            str: The internal acceleration domain. e.g. `cn-hangzhou`, `cn-zhangjiakou`
        r  ry   c                 S   s:   zt j| |d}|  W |S  t jjy   d }Y |S w )Nrx   )r  r   raise_for_statusr  r  )r  ry   r  r   r   r   send_request
  s   
z>HubApi._get_internal_acceleration_domain.<locals>.send_requestz&/api/v1/repos/internalAccelerationInfo)r  ry   r   Nrx  ZInternalRegionQueryAddressrx   )r+  floatrl   r   r  r   )r   rt  rv  Zinternal_urlZinternal_info_responseZ	region_idZ
query_addrZdomain_responser   r   r   !_get_internal_acceleration_domain
  s   	
z(HubApi._get_internal_acceleration_domaindelete_patternsc                C   s  |t vrtd| |stdt|tr|g}t }|s"| j}|du r*td| | j}|t	krH| j
||p:td||d}dd |D }	nc|tkrg }	d	}
d
}	 z| j||pZtd|
||d}W n ty } ztd| dt|  W Y d}~n#d}~ww |D ]}|d dkr|	|d  qt||k rn|
d	7 }
qSn
td| dt  g }|	D ]}|D ]}t||r||  nqqg g }}|D ]}za|t	kr|d\}}| d| d| d}|pt|d}n&|tkr|d\}}| d| d| d}d|i}n
td| dt  | jj||||d}t| | }t| || W q tyS } z|| td| dt|  W Y d}~qd}~ww ||t|dS )a=  
        Delete files in batch using glob (wildcard) patterns, e.g. '*.py', 'data/*.csv', 'foo*', etc.

        Example:
            # Delete all Python and Markdown files in a model repo
            api.delete_files(
                repo_id='your_username/your_model',
                repo_type=REPO_TYPE_MODEL,
                delete_patterns=['*.py', '*.md']
            )

            # Delete all CSV files in the data/ directory of a dataset repo
            api.delete_files(
                repo_id='your_username/your_dataset',
                repo_type=REPO_TYPE_DATASET,
                delete_patterns='data/*.csv'
            )

        Args:
            repo_id (str): 'owner/repo_name' or 'owner/dataset_name', e.g. 'Koko/my_model'
            repo_type (str): REPO_TYPE_MODEL or REPO_TYPE_DATASET
            delete_patterns (str or List[str]): List of glob patterns, e.g. '*.py', 'data/*.csv', 'foo*'
            revision (str, optional): Branch or tag name
            endpoint (str, optional): API endpoint
        Returns:
            dict: Deletion result
        zUnsupported repo_type: zdelete_patterns cannot be emptyNr   T)r   r   rl   rM  c                 S   rQ  r   r   )r  r"  r   r   r   r#  
  rV  z'HubApi.delete_files.<locals>.<listcomp>ro   r}  )r   r   r   r?  r@  rl   zGet dataset: z file list failed, message: ry  treer   r  r   r   z/file)rS  r  r  z/repor  rE  zFailed to delete r  )deleted_filesfailed_filesZtotal_files)rU   r   r*  r+  rz   r   rl   r   r|   rT   r   rO   rS   r   rN   r   r   r   ro  rf  fnmatchr   r}   r   r?   r   r@   )r   r   r   ry  r   rl   r   r|   rp  
file_pathsr?  r@  Zdataset_filesr  Zfile_info_dZ	to_deleter   Zdelete_patternr{  r|  ownerr  r  rF  rs  r   rJ  r   r   r   delete_files
  s   "





&zHubApi.delete_files)Fr   )NNNr  )ro   r>  N)NF)NFN)FN)Tro   r}  N)ro   )rs  )j__name__
__module____qualname____doc__r   r   r   r+  r   r   rN  r   r   r4   ZPUBLICr3   Z	APACHE_V2ra  r   r   r   r   rO   r   r   rC   r   rB   r   rT   r   r  r   r	  staticmethodr  rP   r   r=  r  rC  rL  r   rO  rY  rW  r^  rg  rh  r   rd  ri  r   rr  r2   rv  r  r|  r  r   r   r   r  r  rI  r  rZ   ZREUSE_DATASET_IF_EXISTSr  rN   r  r  r  r  r  r  r  r  r  r  r  r   r  r5   rF   r  r   rg   rf   r  r   r\  r	   r@  r   rO  r9  r
   r   r[  tuplerK  r  rw  rx  r  r   r   r   r   rk   a   s   
!
2
k
U
.
-
#
$

'
:
	

 
+
E



"
G


 



;

(
$
A
8
"

#<
%


"




!	

s	

~
	

 
	

 @	

e
?
9
?,
rk   c                   @   s   e Zd ZeeZdZdZdZdZ	dZ
edd Zedefdd	Zed
d Zedd ZedefddZededefddZedeeef fddZedee fddZeddeeedf defddZdS )rz   r   Z	git_tokenuserr}   Fc                   C   s   t jtjdd d S )NT)r  )r   makedirsrz   path_credentialr   r   r   r   make_sure_credential_path_exist?  s   z0ModelScopeConfig.make_sure_credential_path_existc                 C   sR   t   ttjt jt jd}t	| | W d    d S 1 s"w   Y  d S )Nzwb+)
rz   r  r  r   r   r  r  COOKIES_FILE_NAMEpickler  )r   r"  r   r   r   r   C  s   
"zModelScopeConfig.save_cookiesc                  C   s   t jtjtj} t j| rMt| d1}t	|}|D ]}|j
dkr;| r;tjs;dt_td  W d    d S q|W  d    S 1 sHw   Y  d S )NrY  r   TzKNot logged-in, you can login for uploadingor accessing controlled entities.)r   r   r  rz   r  r  r'  r  r  loadr   
is_expiredcookie_expired_warningr   r  )Zcookies_pathr"  r   cookier   r   r   r   K  s$   


	 
zModelScopeConfig.get_cookiesc                  C   s   t jtjtj} d}t j| r2t| d}t|	 
 dd}|W  d    S 1 s-w   Y  |dks<t|dkrbtt j}t  t| d}|| W d    |S 1 s]w   Y  |S )Nr   rY  r  r      w+)r   r   r  rz   r  USER_SESSION_ID_FILE_NAMEr'  r  r+  readliner   rf  rl  rm  rn  r  r  )Zsession_pathZ
session_idr"  Zwfr   r   r   get_user_session_id\  s$   
 
z$ModelScopeConfig.get_user_session_idr   c                 C   sP   t   ttjt jt jd}||  W d    d S 1 s!w   Y  d S )Nr  )	rz   r  r  r   r   r  r  GIT_TOKEN_FILE_NAMEr  r   r"  r   r   r   r   m  s   
"zModelScopeConfig.save_tokenr  
user_emailc                 C   sX   t   ttjt jt jd}|d| |f  W d    d S 1 s%w   Y  d S )Nr  z%s:%s)	rz   r  r  r   r   r  r  USER_INFO_FILE_NAMEr  )r  r  r"  r   r   r   r   u  s   
"zModelScopeConfig.save_user_infor   c                  C   s~   z4t tjtjtjddd} |  }|dd |dd fW  d    W S 1 s-w   Y  W dS  t	y>   Y dS w )Nr   r  r  :r   ro   r   )
r  r   r   r  rz   r  r  rr   r   FileNotFoundError)r"  r  r   r   r   get_user_info}  s"   
$	zModelScopeConfig.get_user_infoc                  C   sh   d} z't tjtjtjddd}| } W d   W | S 1 s"w   Y  W | S  ty3   Y | S w )z
        Get token or None if not existent.

        Returns:
            `str` or `None`: The token, `None` if it doesn't exist.

        Nr   r  r  )	r  r   r   r  rz   r  r  rr   r  r  r   r   r   	get_token  s&   	

zModelScopeConfig.get_tokenN
user_agentc                 C   s   d}t tjv rtjt  }d}ttjv rtjt }ddlm} d|t t	 t t
 ||f }t| trH|dddd |  D  7 }|S t| trS|d|  7 }|S )	a  Formats a user-agent string with basic info about a request.

        Args:
            user_agent (`str`, `dict`, *optional*):
                The user agent info in the form of a dictionary or a single string.

        Returns:
            The formatted user-agent string.
        Zcustomunknownr   )__version__zSmodelscope/%s; python/%s; session_id/%s; platform/%s; processor/%s; env/%s; user/%sz; c                 s   s"    | ]\}}| d | V  qdS )r   Nr   r  r   r   r   	<genexpr>  s     z2ModelScopeConfig.get_user_agent.<locals>.<genexpr>)r    r   r   r!   Z
modelscoper  platformpython_versionrz   r  	processorr*  r  r  r  r+  )r  envr  r  Zuar   r   r   r{     s,   




	 
zModelScopeConfig.get_user_agentr  )r  r  r  r   r"   r  r  r  r  r  r  r  r  r   r   r   r  r+  r   r   r   r  r   r  r   r
   r{   r   r   r   r   rz   7  s2    


$rz   c                   @   s   e Zd ZdZeeeeefde	de	de	de	de	f
ddZ
dddZdeeef fddZdeeeeef ded	efddZdeeeef  ded	d
fddZd
S )r   a  
    Check the files and folders to be uploaded.

    Args:
        max_file_count (int): The maximum number of files to be uploaded. Default to `UPLOAD_MAX_FILE_COUNT`.
        max_file_count_in_dir (int): The maximum number of files in a directory.
            Default to `UPLOAD_MAX_FILE_COUNT_IN_DIR`.
        max_file_size (int): The maximum size of a single file in bytes. Default to `UPLOAD_MAX_FILE_SIZE`.
        size_threshold_to_enforce_lfs (int): The size threshold to enforce LFS in bytes.
            Files larger than this size will be enforced to be uploaded via LFS.
            Default to `UPLOAD_SIZE_THRESHOLD_TO_ENFORCE_LFS`.
        normal_file_size_total_limit (int): The total size limit of normal files in bytes.
            Default to `UPLOAD_NORMAL_FILE_SIZE_TOTAL_LIMIT`.

    Examples:
        >>> from modelscope.hub.api import UploadingCheck
        >>> upload_checker = UploadingCheck()
        >>> upload_checker.check_file('/path/to/your/file.txt')
        >>> upload_checker.check_folder('/path/to/your/folder')
        >>> is_lfs = upload_checker.is_lfs('/path/to/your/file.txt', repo_type='model')
        >>> print(f'Is LFS: {is_lfs}')
    max_file_countmax_file_count_in_dirmax_file_sizesize_threshold_to_enforce_lfsnormal_file_size_total_limitc                 C   s"   || _ || _|| _|| _|| _d S r  )r  r  r  r  r  )r   r  r  r  r  r  r   r   r   r     s
   
zUploadingCheck.__init__r   Nc                 C   sl   t |ttfrtj|std| dt|}|| jkr4t	
d| jd  dt|d d d dS dS )	a  
        Check a single file to be uploaded.

        Args:
            file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.

        Raises:
            ValueError: If the file does not exist or exceeds the size limit.
        File  does not existzFile exceeds size limit:    @z	 GB, got     GBN)r*  r+  r   r   r   r'  r   r`   r  r   r   rb  )r   r&  r'  r   r   r   r7    s   

zUploadingCheck.check_filerD  c                 C   s  d}d}t |trt|}| D ][}| r>|d7 }t|}|| jkr=td| d| jd  ddt	|d d d q|
 rl|d7 }| |\}}|| | jkrdtd	| d
||  d| j ||7 }||7 }q|| jkr}td| d| j ||fS )z
        Check a folder to be uploaded.

        Args:
            folder_path (Union[str, Path]): The folder path to be checked.

        Raises:
            ValueError: If the folder does not exist or exceeds the file count limit.
        r   ro   r  z exceeds size limit: r  r  zgot r  z
Directory z
 contains z items and exceeds limit: zTotal file count z and exceeds limit: )r*  r+  r   iterdirrc  r`   r  r   r   rb  rk  ri  r  r   r  )r   rD  Z
file_countZ	dir_countr]  Z	item_sizeZsub_file_countZsub_dir_countr   r   r   ri    s4   



zUploadingCheck.check_folderr&  r   c                 C   s   d}t |ttfr;t|}| std| d|tkr%|jtvr$d}n|tkr1|jt	vr0d}n
td| dt
 t|}|| jkpE|S )aj  
        Check if a file should be uploaded via LFS.

        Args:
            file_path_or_obj (Union[str, Path, bytes, BinaryIO]): The file path or file-like object to be checked.
            repo_type (str): The repo type, either `model` or `dataset`.

        Returns:
            bool: True if the file should be uploaded via LFS, False otherwise.
        Tr  r  Fr   r  )r*  r+  r   r'  r   rT   r<  rd   rS   rb   rU   r`   r  )r   r&  r   Zhit_lfs_suffixr'  r   r   r   r:  $  s    

zUploadingCheck.is_lfsr%  c                    sT    fdd|D }t dd |D }|jkr(td|d  djd  ddS )	af  
        Check a list of normal files to be uploaded.

        Args:
            file_path_list (List[Union[str, Path]]): The list of file paths to be checked.
            repo_type (str): The repo type, either `model` or `dataset`.

        Raises:
            ValueError: If the total size of normal files exceeds the limit.

        Returns: None
        c                    s   g | ]
} | s|qS r   )r:  r  r]  r   r   r   r   r#  P  r$  z5UploadingCheck.check_normal_files.<locals>.<listcomp>c                 S   s   g | ]}t |qS r   )r`   r  r   r   r   r#  Q  rV  zTotal size of non-lfs files i   zMB and exceeds limit: MBN)sumr  r   )r   r%  r   Znormal_file_listr  r   r  r   r8  C  s   

z!UploadingCheck.check_normal_files)r   N)r  r  r  r  r,   r-   r.   r0   r/   ra  r   r7  r   r+  r   ri  r\  r	   rN  r:  r   r8  r   r   r   r   r     s,    

"&&r   )r4  r}  r   r5  r   r  r  r  r.  r  r  rl  r%  collectionsr   httpr   http.cookiejarr   Zos.pathr   pathlibr   typingr   r	   r
   r   r   r   r   r   r   r   r   r  r   Zrequests.adaptersr   r   Zrequests.exceptionsr   r  r   Zmodelscope.hub.constantsr   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   Zmodelscope.hub.errorsr7   r8   r9   r:   r;   r<   r=   r>   r?   r@   Zmodelscope.hub.gitrA   Zmodelscope.hub.inforB   rC   r  rE   Zmodelscope.hub.utils.aigcrF   Zmodelscope.hub.utils.utilsrG   rH   rI   rJ   rK   rL   rM   Zmodelscope.utils.constantrN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   Zmodelscope.utils.file_utilsr_   r`   Zmodelscope.utils.loggerra   Zmodelscope.utils.repo_utilsrb   rc   rd   re   rf   rg   rh   ri   Zmodelscope.utils.thread_utilsrj   r   rk   rz   r   r   r   r   r   <module>   s   (0$L
(                     k 