o
    [+ iPf                     @   s  d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZmZ ddlmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ejZejZejZejZdd Zdd Zdd Zdd Zdd Z dd Z!e!dd Z"e!dd Z#dd Z$dd Z%d d! Z&d"d# Z'd$e(d%e(fd&d'Z)d(d) Z*e!d*d+ Z+d,d- Z,d.d/ Z-d0d1 Z.d2d3 Z/d4d5 Z0d6d7 Z1dMd9e2d:e3fd;d<Z4dNd=d>Z5d?d@ Z6dAdB Z7edCe8 dDdEdF Z9dGZ:e:dH dH Z;e;fdIdJZ<dOdKdLZ=dS )Pu^   
本文件实现了请求hub

Authors: xiangyiqing(xiangyiqing@baidu.com)
Date:    2023/07/24
    N)quote)	ErrorEnum)configlog)err_resp)gen_ISO_format_datestr)file_to_base64thread_executor)$create_sha256_file_and_encode_base64)VERSION)RequestErrorc              	   C   sz   t tD ],}ztj| |||td}| W   S  tjjy0   d|j	}|j
}t| Y qw ttjj|dd |S )z
    request aistudio hub
    headersdatatimeout-Response body does not contain valid json: {}N  )rangeCONNECTION_RETRY_TIMESrequestsrequestCONNECTION_TIMEOUTjson
exceptionsJSONDecodeErrorformattextstatus_coder   debugr   r   INTERNAL_ERRORcode)methodurlr   r   _responseerr_msgbiz_code r'   e/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/aistudio_sdk/requests/hub.py_request_aistudio_hub%   s   

r)   c                  K      t | d d}| d dtjdtjdtj}dd |  D }t	
d|  t	
d	|  t|}td
|||}|S )u'   
    请求AIStudio hub: 模型库
    tokenr+   {}{}STUDIO_MODEL_API_URL_PREFIXdefaultc                 S      i | ]\}}||qS r'   r'   .0kvr'   r'   r(   
<dictcomp>C       z(request_aistudio_hub.<locals>.<dictcomp>zrequest_aistudio_hub url: zrequest_aistudio_hub body: POST)_header_fillpopr   osgetenvr   #STUDIO_MODEL_API_URL_PREFIX_DEFAULTZHUB_URLitemsr   r   r   dumpsr)   kwargsr   r"   bodypayloadrespr'   r'   r(   request_aistudio_hub7   s   

rE   c                  K   r*   )u(   
    请求AIStudio hub：应用库
    r+   r,   r-   r.   r/   c                 S   r1   r'   r'   r2   r'   r'   r(   r6   Y   r7   z0request_aistudio_app_service.<locals>.<dictcomp>z"request_aistudio_app_service url: z#request_aistudio_app_service body: r8   )r9   r:   r   r;   r<   r   r=   ZAPP_SERVICE_URLr>   r   r   r   r?   r)   r@   r'   r'   r(   request_aistudio_app_serviceL   s   

rF   c                  K   s   t | d d}dtjdtjdtj}|dt| d dd	 d
 }d}zd}tj	|||t
d}| W S  tjjyG   d|}|j}Y nw ttjj|dd |S )u3   
    请求AIStudio hub 查看repo可见权限
    r+   r,   r-   r.   r/   z?repoId=ZrepoId safez&authorization=1GET)r   r   r   Nr   )r9   r   r;   r<   r   r=   ZHUB_URL_VISIBLE_CHECKr   r   r   r   r   r   r   r   r   r   r   r    )rA   r   r"   r!   r%   r$   r&   r'   r'   r(   request_aistudio_repo_visibleb   s*   




rK   c                 C   s   t tD ]\}t }|j| |||td}|  |jdvr[t	d|j
 d|j  td| d|  |jdkr<dnd	}t| d
krGtjjntjj|j dd | |jd  S |   S dS )z
    request gitea
    r         z
response:  z'potential network problem while request:  u   [仓库或分支不存在]rG   rJ   Nr   r&   )r   r   r   Sessionr   r   closer   r   r   r   infor   r   GITEA_DOWNLOAD_FILE_FAILEDr    GITEA_UPLOAD_FILE_FAILEDcontentdecoder   )r!   r"   r   r   r#   sessionr$   Z	extra_msgr'   r'   r(   _request_gitea}   s"   

r[   c                    s    fdd}|S )z
    time cost decorator
    c                     sB   t   } | i |}t   }|| }t j d|dd |S )Nz done, time cost: .2fs)timeprint__name__)argsrA   
start_timeresultend_timeZelapsed_timefuncr'   r(   wrapper   s   z!timing_decorator.<locals>.wrapperr'   )rf   rg   r'   re   r(   timing_decorator   s   rh   c                 C   sV   t  }|j| |||dtd}|  |jdvr'ttjj	|j
dd |jdS | S )z
    _upload proc
    T)r   r   streamr   rL   Nr   rR   )r   rS   r   CONNECTION_TIMEOUT_UPLOADrT   r   r   r   rW   r    rX   r   )r!   r"   r   r   rZ   r$   r'   r'   r(   _upload   s   
rk   c                 C   s0  t jd| d|td}|jdkri }n5|jdkr:z| d }W n t jjy/   |j }Y nw t	t
jj||j}nt	t
jjd|j |j}t|jdd	}d
}tj|ddddd}t|d}	||D ]}
|t|
 |	|
 qgW d   n1 sw   Y  |  |d	kr|j|krtd |S )z
    Params
        :url: http url
        :download_path: download path
        :headers: headers
    Returns
        file
    rJ   T)ri   r   r   rM   rQ   messagez Download failed, response code: zcontent-lengthr   i  2   ZiBzDownloading file)totalZncolsunitZ
unit_scaleZdescwbNzERROR, something went wrong)r   r   CONNECTION_TIMEOUT_DOWNLOADr   r   r   r   rX   rY   r   r   ZFILE_NOT_FOUNDr    rV   intr   gettqdmopeniter_contentupdatelenwriterT   nr_   )r"   download_pathr   r$   retrl   
total_size
block_sizeprogress_barfiler   r'   r'   r(   	_download   sD   


r   c                 C   s   t |d}t| ||}|S )u*   
    请求AIStudio gitea文件下载
    r,   )r9   r   )r"   r{   r+   r   resr'   r'   r(   request_aistudio_git_download   s   
r   c           	   
   C   s   |  dt |dd dt |dd dt |dd }|dkr(|dt |dd 7 }t|d}td	||d}td
| d|  |S )u   
    请求AIStudio gitea 文件info
    GET /api/v1/repos/{owner}/{repo}/contents/{filepath} 返回的文件的数据、大小、编码等metadata信息+文件内容，或者文件夹中的文件列表
    /api/v1/repos/rG   rH   /
/contents/Zmaster?ref=r,   rJ   z...result of GET /contents/: )r   r9   r[   r   r   )		call_host	user_name	repo_name	file_pathrevisionr+   r"   r   r   r'   r'   r(   request_aistudio_git_file_info   s   


r   c              
   C   s   t |d}|  dt|dd dt|dd dt|dd }dd|igi}t|}	td|||	d	}
td
| d|
  d|
v rE|
}|S d|
vsU|
d rUd|
d d vrdttj	j
t|
dd }|S d|
d d d i}|S )u1   
    请求AIStudio gitea 确认文件类型
    r,   r   rG   rH   /preupload/filespathr8   r   ...result of POST /preupload: r   
error_codelfsr   Nr   is_lfs)r9   r   r   r?   r[   r   r   r   r   GITEA_FAILEDr    str)r   r   r   r   path_in_repor+   r   r"   paramsrC   rc   r   r'   r'   r(   request_aistudio_git_file_type   s(   
2
	 r   c              
   C   s   t |d}|  dt|dd dt|dd dt|dd }ddd	 |D i}t|}	td
|||	d}
td| d|
  d|
v rGtdd|
vsRt|
d t	sVtdi }|
d D ]}|
d}|
dd}|ro|||< q\|S )u   
        批量请求 AIStudio gitea 确认多个文件是否为 LFS 类型

        :param path_list: List[str]，每个元素是 repo 内的相对路径
        :return: Dict[str, bool]，key 是路径，value 是是否为 LFS
        r,   r   rG   rH   r   r   r   c                 S   s   g | ]}d |iqS )r   r'   )r3   r   r'   r'   r(   
<listcomp>)  s    z3request_aistudio_git_files_type.<locals>.<listcomp>r8   r   r   r   r   z-preupload fail, there is error_code in resultz#preupload fail, wrong result formatr   r   F)r9   r   r   r?   r[   r   r   
ValueError
isinstancelistrs   )r   r   r   r   Zpath_in_repo_listr+   r   r"   r   rC   rc   Zlfs_map	file_infor   r   r'   r'   r(   request_aistudio_git_files_type  s2   





r   upload_sectionreturnc              	   C   s\   |  di }|r,| dr,| d| d| d| d| d| d| dd	S i S )
u  
    解析sts_token

    "upload": {
        "href": "https://some-download.com",
        "header": {
            "Key": "value"
        },
        "sts_token": {
            "bosHost":""
            "bucketName": "",
            "key":"",
            "accessKeyId":"",
            "secretAccessKey":"",
            "sessionToken":"",
            "createTime":"",
            "expiration":""
            }
        "expires_at": "2016-11-10T15:29:07Z"
    }
    	sts_tokenZaccessKeyIdZbosHostZ
bucketNamekeyZsecretAccessKeyZsessionToken
expiration)Zbos_hostZbucket_namer   Zaccess_key_idZsecret_access_keyZsession_tokenr   )rs   )r   r   r'   r'   r(   _parse_sts_token>  s   	r   c                 C   sF  ddd}t ||d}|  dt|dd dt|dd d}	d	||d
gddgdd| idd}t|}
td|	||
}td|  d|v rM|}|S d|vsU|d sdttj	j
t|dd }|S |d d }d|v rvd	|d v rvdndd|v r|d d	 d ndd|v rt|d d	 ni d|v r|d d d ndd}|S )uL   
    请求AIStudio gitea 申请上传LFS文件.
    只支持单文件
    z+application/vnd.git-lfs+json; charset=utf-8application/vnd.git-lfs+jsonContent-TypeAcceptr   r+   r   rG   rH   z.git/info/lfs/objects/batchuploadoidsizezlfs-standalone-filebasicnamezrefs/heads/sha256)	operationobjectsZ	transfersrefZ	hash_algor8   z...result of POST /batch: r   r   Nr   r   actionsTFhrefverify)r   Zupload_hrefr   Zverify_href)r9   r   r   r?   r[   r   r   r   r   r   r    r   r   )r   r   r   r   	file_sizer   r+   r   r   r"   rC   rc   r   tmpr'   r'   r(   "request_aistudio_git_upload_accessc  sD   &

r   c              	   C   sN   t |d}tjd| ||tdd}W d   n1 sw   Y  d|jd iS )   
    上传LFS文件到bos
    rbPUTT)r   r   r   ri   NzContent-Md5)ru   r   r   rj   r   )r"   path_or_fileobjr   r   r$   r'   r'   r(   _lfs_upload  s   r   c                 C   s    ddi}t |dd}t| ||S )r   r   zapplication/octet-streamrG   r   )r9   r   )r"   r   r   r   r'   r'   r(   request_bos_upload  s   r   c                 C   s8   d| v r| d t jjkrdS | rd| vrdS | d }|S )u   
    解析info_res
    r   rG   sha)r   SUCCESSr    )info_resold_shar'   r'   r(   get_exist_file_old_sha  s   r   c	              
   C   s   t | |||||}	t|	}
|
dkrd}nd}t|d}|  dt|dd dt|dd dt|dd }||t||d	t t d
|d}|dkrM|
|d< t|}t||||}|S )C   
    请求AIStudio gitea 上传LFS指针文件（到仓库）
    rG   r8   r   r,   r   rH   r   r   Tauthor	committer)branch
new_branchrX   
lfsPointerdatesrl   r   )	r   r   r9   r   r
   r   r   r?   r[   )r   r   r   r   commit_messager   r   r   r+   r   r   r!   r   r"   r   rC   r   r'   r'   r(   #request_aistudio_git_upload_pointer  s6   




r   c              
   C   s   t | |||||}t|}	|	dkrd}
nd}
t|d}|  dt|dd dt|dd dt|dd }||t||d	d
}|
dkrF|	|d< t|}t|
|||}|S )r   rG   r8   r   r,   r   rH   r   r   T)r   r   rX   r   r   )r   r   r9   r   r
   r   r?   r[   )r   r   r   r   r   r   r   r+   r   r   r!   r   r"   r   rC   r   r'   r'   r(   !request_single_git_upload_pointer  s.   




r   c              
   C   s   t | |||||}t|}	|	dkrd}
nd}
|  dt|dd dt|dd dt|dd }t|d}t|}|||d	t t d
|d}|
dkrN|	|d< t|}t|
|||}|S )@   
    请求AIStudio gitea 上传普通文件（到仓库）
    rG   r8   r   r   rH   r   r   r,   Fr   )r   r   rX   r   r   rl   r   )	r   r   r   r9   r   r   r   r?   rk   )r   r   r   r   r   r   r   r+   r   r   r!   r"   r   base64_datar   rC   r   r'   r'   r(   "request_aistudio_git_upload_common  s8   




r   c              
   C   s   t | |||||}t|}|dkrd}	nd}	|  dt|dd dt|dd dt|dd }
t|d}t|}|d	d
}|	dkrE||d< t|}t|	|
||}|S )r   rG   r8   r   r   rH   r   r   r,   F)rX   r   r   )r   r   r   r9   r   r   r?   rk   )r   r   r   r   r   r   r+   r   r   r!   r"   r   r   r   rC   r   r'   r'   r(    request_single_git_upload_common0  s,   




r   rG   r   r   c                 C   st   ddd}||d}t ||d}tjd| ||t|d}td|j  |jdvr6t	t
jj|j|jd	S | S )
zp
    param
        call_host: verify url
        oid: sha256, without sha256prefix
        size: file size

    r   r   r   r,   r8   )r   r   r   z...result of POST /verify: rL   rR   )r9   r   r   r   r?   r   r   r   r   r   r   rW   r    )r   r   r   r+   r   r   headerr   r'   r'   r(    request_aistudio_verify_lfs_fileX  s   	
r   c                 C   sF   |rd| }n
dt jddd }d|ttd}| r!||  |S )u   
    填充header
    ztoken ZAISTUDIO_ACCESS_TOKENrG   r/   zapplication/json)r   AuthorizationzSDK-Version)r;   r<   r   r   rw   )r   r+   authr   r'   r'   r(   r9   s  s   
r9   c                 C   sb   t jdtjd}| d|  d| d| }t|d}tj||d}|jdkr/d	| d
fS dS )uH   检查目标文件是否存在，返回 (exists: bool, sha: str or None)STUDIO_GIT_HOSTr/   r   r   r   r,   )r   rM   Tr   )FN)	r;   r<   r   STUDIO_GIT_HOST_DEFAULTr9   r   rs   r   r   )repo_idr   r   r+   hostr"   r   rD   r'   r'   r(   file_exists_and_sha  s   

r   c                 C   s   d|  d| dS )u.   构造 LFS pointer 文件内容（纯文本）z6version https://git-lfs.github.com/spec/v1
oid sha256:z
size 
r'   )r   r   r'   r'   r(   get_lfs_pointer_content  s
   r   F)Zdisable_tqdmmax_workersc              
   C   s
  | \}}}}zt ||||\}}	W n ty, }
 zt| d W Y d}
~
dS d}
~
ww |r=tj|}t|p9||}n5t|d}|	 }W d   n1 sQw   Y  zt
|}|d}W n tyq   td| dw ||||rydndd	}|	r|	|d
< |S )z
    prepare body data
    z request git error,skipNr   zutf-8u   ❌ Non-UTF8 content in z-. Consider LFS or base64 with encoding field.rw   create)r   r   rX   r   r   )r   	Exceptionr   errorr;   r   getsizer
   ru   readbase64	b64encoderY   UnicodeDecodeErrorr   )qr   r   r+   r   
local_pathr   sha256_inputexistsr   er   Zcontent_b64fZcontent_bytesZcontent_strentryr'   r'   r(   prepare_entry  s8   


r   rM      c                 C   s   g }g }d}d}| D ]F}| dd}||kr#td|d d dd|| |ks6|d ttj dd	krC|| |g}|}d}q
|| ||7 }|d7 }q
|rX|| |S )
u   
    将 files 拆分为多个分组，每组最大总大小不超过 max_bytes,每组个数不超过1k
    要求每个 file 是一个 dict，包含 'size' 字段（单位字节）。
    r   r   u!   单个文件超过最大限制：r   r\   MB   ZMAX_COMMIT_FILE_COUNT500)rs   r   rr   r;   environappend)r   Z	max_byteschunksZcurrent_chunkZcurrent_sizeZcurrent_countr   r   r'   r'   r(   split_files_by_size  s*   



r   c                 C   s  |du r	ddd}|du r|}t d t dt   t||||d}dd	 |D }t|}	t|	d
kr<t d tjdt	j
d}
t|d}|
 d| d}t|	d
dD ]i\}}t|	d
krc|n| d| dt|	 d}|||||d}tj|||d}|jd dkrt|j td| d qW|D ]}|D ]\}}}}|d |kr| |df qqtd| d|j  t|j qWdS )z
    commit
    NzAuto Commitzauto@example.com)r   emailzcalculate fileszcpu:)r   r   r+   c                 S   s   g | ]}|d ur|qS )Nr'   )r3   itemr'   r'   r(   r     s    z commit_files.<locals>.<listcomp>r   z'files will be commited in multi batchesr   r/   r,   r   z	/contents)startz (part r   ))r   rl   r   r   r   )r   r   d      u   ✅ Commit part z successful!r   zcommit failu   ❌ Commit part z	 failed: )r   rU   r   r;   	cpu_countr   r   rx   r<   r   r   r9   	enumerater   postr   r_   r   r   )Zlog_listr   r   r   Z
file_quadsr+   r   r   r   Zfile_chunksr   r   r"   ichunkZcommit_message_currentZchunk_payloadrD   r   r   r   r   r   r'   r'   r(   commit_files  sJ   




r  )rG   )NrG   )NN)>__doc__r   r;   r^   rt   r   r   urllib.parser   Zaistudio_sdk.constant.err_coder   Zaistudio_sdkr   r   Zaistudio_sdk.utils.utilr   r   r   r	   r
   Zaistudio_sdk.constant.versionr   Zaistudio_sdk.errorsr   r   r   rq   rj   r)   rE   rF   rK   r[   rh   rk   r   r   r   r   r   dictr   r   r   r   r   r   r   r   r   r   rr   r   r9   r   r   r  r   ZMAX_PAYLOAD_MBZMAX_PAYLOAD_BYTESr   r  r'   r'   r'   r(   <module>   sl   

.	#%4

	''*(

" 