o
    [+ i;                     @   s  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
mZmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZmZmZmZmZmZmZ dd	lmZm Z  dd
l!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl!m)Z) ddl*mZm+Z+ ddl,m-Z- dd Z.dddddededf	dee/ dee0 dee/ deee
e/ e/f  deee
e/ e/f  de1de/dee/ de/de/fdd Z2dedddddedd!	de/dee/ dee/ d"ee/e(df dee0 dee/ deee
e/ e/f  deee
e/ e/f  de1de/fd#d$Z3d%d& Z4deddedfd'e
e5 d(e$d)e/de/dee/ dee/ deee
e/ e/f  deee
e/ e/f  de1de/fd*d+Z6d,d- Z7d.e/fd/d0Z8d1ee/e
e/ f fd2d3Z9d1e
e/ fd4d5Z:edfd6e/dee/ de/de
e5 e5ffd7d8Z;dd9de/d:e/dee/ fd;d<Z<dS )=u8   
本文件实现了星河社区git仓库的下载功能
    N)Class)DictListOptionalUnion)log)$create_temporary_directory_and_cachedownload_fileget_git_info)InvalidParameterraise_on_error)REPO_TYPE_SUPPORTREPO_TYPE_DATASETREPO_TYPE_MODELSTUDIO_GIT_HOST_DEFAULTDEFAULT_MODEL_REVISIONDEFAULT_DATASET_REVISIONDEFAULT_MAX_WORKERS)thread_executorget_model_masked_directory)quote)ModelFileSystemCache)_header_fill)Path)urlparse   )r   NotExistError)post_repo_statisticc                  C   s   t jddds=t jdt dd} |  d}t j|r?t|d}|  t jd< W d   dS 1 s6w   Y  dS dS dS )	u   初始化函数，从本地磁盘加载AI Studio认证令牌。

    Args:
        无参数。

    Returns:
        无返回值。
    ZAISTUDIO_ACCESS_TOKEN defaultZAISTUDIO_CACHE_HOMEHOMEz/.cache/aistudio/.auth/tokenrN)osgetenvpathexistsopenreadstripenviron)Z
cache_homeZtoken_file_pathfile r,   j/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/aistudio_sdk/snapshot_download.pyinit#   s   
"r.   Frevisionlocal_files_only	local_dirallow_patternsignore_patternsmax_workersrepo_id	repo_typetokenreturnc	                 C   s   t   |s	td|tvrtd| dt | du r#|tkr!tnt} d|i}	zt|| |	 W n tyI }
 zt	d|
  W Y d}
~
nd}
~
ww t
||| ||||||d	S )preparez*Please provide a valid model_id or repo_idInvalid repo type: z, only support: Nrepozrequest.dot.fail: )r6   r/   r0   r1   r3   r2   r4   r7   )r.   
ValueErrorr   r   r   r   r   	Exceptionr   debug_snapshot_download)r/   r0   r1   r2   r3   r4   r5   r6   r7   actioner,   r,   r-   snapshot_download7   s6   rB   )	r6   r/   	cache_dirr0   r1   r2   r3   r4   r7   rC   c       	         C   s  |st }|tvrtd|tf t| ||d\}
}tjtdt }|r<t	|j
dkr1tdtd|  | S dtt i}|t ksL|tkr|rUtj|}n$|retjj|g| dR  }ntjj||t krod	nd
g| dR  }td|  t| ||	d\}}t|||
| ||||||	d
 d| v rt|| }tj|rtd n1td| d ztjtj||dd W n ty   td| dtj| d Y nw |j|d | }|S )z
    download repo
    z'Invalid repo type: %s, only support: %s)r1   r6   ZAISTUDIO_CACHEr   zCannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z6We can not confirm the cached file is for revision: %szsnapshot-identifier/modelsZdatasetz,Downloading Model from remote to directory: )model_idr/   r7   )

repo_filescachetemporary_cache_dirr5   r6   r/   r3   r2   r4   r7   .z3Target directory already exists, skipping creation.zCreating symbolic link [z].T)target_is_directoryzFailed to create symbolic link z for )revision_info)r   r   r   r   r#   r%   
expanduserr$   get_default_aistudio_cache_dirlencached_filesr<   r   warnZget_root_locationstruuiduuid4r   abspathjoinsplitinfoget_model_files_download_file_listsr   r&   symlinkOSErrorZsave_model_version)r5   r6   r/   rC   r0   r1   r2   r3   r4   r7   rI   rH   Zsystem_cacheheaders	directoryrG   Zrevision_detailZmasked_directoryZcache_root_pathr,   r,   r-   r?   a   s   



r?   c                  C   s   t jt ddd} | S )z/
    default base dir: '~/.cache/aistudio'
    z.cacheZaistudioZhub)r#   r%   rM   r   homejoinpath)Zdefault_cache_dirr,   r,   r-   rN      s   rN   rG   rH   rI   c
                    s  t |}t |}g }
| D ]Ld dkrqz$|r%tfdd|D r%W q|dur8|r8tfdd|D s8W qW n tyR } ztd|  W Y d}~qd}~ww |
 qt|d	d
 fdd}t|
dkrtdt|
 d ||
 td d d dS dS )z
    download all
    typetreec                    s   g | ]
}t   d  |qS r%   fnmatch.0pattern	repo_filer,   r-   
<listcomp>   s    z(_download_file_lists.<locals>.<listcomp>Nc                 3   s     | ]}t   d  |V  qdS )r%   Nrd   rf   ri   r,   r-   	<genexpr>   s
    
z'_download_file_lists.<locals>.<genexpr>z The file pattern is invalid : %sF)r4   disable_tqdmc              
      s  t | d }  | r!tj| d }td| d dS tkrh| d }d\}}|	 }|	 }tj
dtd}| dt|d	d
 dt|d	d
 dt|d	d
 }dkrg|dtd	d
 7 }nQtkr| d }d\}}|	 }|	 }tj
dtd}| dt|d	d
 dt|d	d
 dt|d	d
 }dkr|dtd	d
 7 }n
td dt t||  dd dS )zdownload each filer%   z
File z8 already in cache with identical hash, skip downloading!NrD   STUDIO_GIT_HOSTr   /api/v1/repos/r   )safez/media/Zmasterz?ref=r:   z, supported types: F)rm   r7   )r
   r&   r#   r%   basenamer   rX   r   rW   r)   r$   r   r   r   r   r   r	   )rj   	file_name	file_pathZ	user_name	repo_nameZgit_hosturl)rH   r5   r6   r/   rI   r7   r,   r-   _download_single_file   sh   








z3_download_file_lists.<locals>._download_single_filer   zGot z files, start to download ...z	Download z 'z' successfully.)	_normalize_patternsanyr=   r   rQ   appendr   rO   rX   )rG   rH   rI   r5   r6   r/   r2   r3   r4   r7   Zfiltered_repo_filesrA   rv   r,   )rH   rj   r5   r6   r/   rI   r7   r-   rZ      s>   	
4rZ   c           	   	   C   s   d}d}g }	 | j |||dd||d}d|v r|d dks.td|d	  d
|d   dS |d d }|| t||k rB	 |S |d7 }q)zget repo metar      TrD   )Zdataset_name	namespacer/   	root_path	recursivepage_number	page_sizeZCode   z;Get dataset file list failed, request_id:                  Z	RequestIdz, message: MessageNZDataZFiles)Zlist_repo_treer   errorextendrO   )	Z_apinameZgroup_or_ownerr/   r~   r   rG   Zfiles_list_treeZcur_repo_filesr,   r,   r-   fetch_repo_files7  s6   	
r   rh   c                 C   s&   zt |  W dS  ty   Y dS w )checkTF)recompileBaseException)rh   r,   r,   r-   _is_valid_regexX  s   
r   patternsc                 C   s*   t | tr| g} | durdd | D } | S )	normalizeNc                 S   s"   g | ]}| d s|n|d qS )rD   *)endswith)rg   itemr,   r,   r-   rk   f  s    z'_normalize_patterns.<locals>.<listcomp>)
isinstancerR   )r   r,   r,   r-   rw   a  s   
rw   c                 C   s0   | durg }| D ]}t |r|| q|S dS )zprocess regexN)r   ry   )r   Zregex_patternsr   r,   r,   r-   _get_valid_regex_patternl  s   
r   rF   c              	   C   sJ  t jdtd}| d|  d| }t|d}tt j|d< d}d}g }d	}	i }
tj	||d
}|j
dv r=| d d }|	r| d|  d| d| d| 	}tj	||d
}|j
dvrct| td| }t| |dkrt|d }t j|j}d|i}
|	dg D ]}|d dv rq|| q|	dd}	|d7 }|	s?||
fS )zList the models files.

    rn   r   ro   z/tags/)r7   zX-Request-IDr   i  T)r]   )r      commitshaz/git/trees/z?recursive=true&page=z
&per_page=zrepo not foundru   ZRevisionrb   r%   )z
.gitignorez.gitattributes	truncatedF)r#   r$   r   r   rR   rS   rT   hexrequestsgetstatus_codejsonprintr   r   r   r%   rq   ry   )rF   r/   r7   ZendpointZtag_pathr]   pageZper_pagefilesr   rL   Ztag_resr%   r"   d
parsed_urlrr   r+   r,   r,   r-   rY   x  s@   

 
rY   r/   filenamec                C   s"   t | |d}dd |D }||v S )aP  Get if the specified file exists

    Args:
        repo_id (`str`): The repo id to use
        filename (`str`): The queried filename, if the file exists in a sub folder,
            please pass <sub-folder-name>/<file-name>
        revision (`Optional[str]`): The repo revision
    Returns:
        The query result in bool value
    r   c                 S   s   g | ]}|d  qS rc   r,   )rg   r+   r,   r,   r-   rk     s    zfile_exists.<locals>.<listcomp>)rY   )r5   r   r/   r   r,   r,   r-   file_exists  s   r   )=__doc__r   rS   re   r#   r   Zsymtabler   typingr   r   r   r   Zaistudio_sdkr   Zaistudio_sdk.file_downloadr   r	   r
   Zaistudio_sdk.errorsr   r   Zaistudio_sdk.configr   r   r   r   r   r   r   Zaistudio_sdk.utils.utilr   r   urllib.parser   Zaistudio_sdk.utils.cachingr   Zaistudio_sdk.requests.hubr   pathlibr   r   errorsr   Zaistudio_sdk.dotr   r.   rR   boolintrB   r?   rN   dictrZ   r   r   rw   r   rY   r   r,   r,   r,   r-   <module>   s  $	

-	

^	

o!	

3