o Zi1 @s@UddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z ddl m Z ddlmZddlmZdd lmZmZmZmZmZdd lmZmZddlZddlZdd lmZmZm Z dd l!m"Z"d dl#m$Z$d dl%m&Z&m'Z'd dl(m)Z)m*Z*m+Z+m,Z,d dl-m.Z.m/Z/d dl0m1Z1m2Z2m3Z3d dl4m5Z5m6Z6m7Z7m8Z8d dl9m:Z:e;dej<Z=e GdddZ>e?ej@ZAeeBd<GdddeAZCGdddej@eCdZDGdddejEjFZGGd d!d!ejEjFZHd"eId#eIfd$d%ZJd&eId#eIfd'd(ZKd)eId*eeLd#efd+d,ZMd-eDd)eId.eId/eNd0eIf d1d2ZOd3d4ZPeDZQdS)5N)deque) ExitStack)deepcopy) dataclassfield)datetime)chain)Path)AnyIteratorNoReturnOptionalUnion)quoteunquote)_DEFAULT_CALLBACK NoOpCallback TqdmCallback) isfilelike) constants)CommitOperationCopyCommitOperationDelete)EntryNotFoundErrorHfHubHTTPErrorRepositoryNotFoundErrorRevisionNotFoundError) hf_hub_urlhttp_get)HfApiLastCommitInfoRepoFile)HFValidationErrorhf_raise_for_status http_backoffhttp_stream_backoff)md5zy (^refs\/convert\/\w+) # `refs/convert/parquet` revisions | (^refs\/pr\/\d+) # PR revisions c@sXeZdZUdZeed<eed<eed<eed<edddZeeed <d efd d Z dS) HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path. repo_typerepo_idrevision path_in_repoNF)defaultrepr _raw_revisionreturncCs~tj|jd|j}|jr|d|jd|jdS|jtj kr4|dt |jd|jdS|d|jdS)N@/) rREPO_TYPES_URL_PREFIXESgetr(r)r.r+rstripr*DEFAULT_REVISION safe_revision)self repo_pathr:U/home/app/Keep/.python/lib/python3.10/site-packages/huggingface_hub/hf_file_system.py unresolve4s  "z"HfFileSystemResolvedPath.unresolve) __name__ __module__ __qualname____doc__str__annotations__rr.r r<r:r:r:r;r'(s r' _cached_basecs(eZdZdZfddZddZZS)_Cacheda Metaclass for caching HfFileSystem instances according to the args. This creates an additional reference to the filesystem, which prevents the filesystem from being garbage collected when all *user* references go away. A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* be made for a filesystem instance to be garbage collected. This is a slightly modified version of `fsspec.spec._Cached` to improve it. In particular in `_tokenize` the pid isn't taken into account for the `fs_token` used to identify cached instances. The `fs_token` logic is also robust to defaults values and the order of the args. Finally new instances reuse the states from sister instances in the main thread. cstj|i|i|_dSN)super__init___cache)clsargskwargs __class__r:r;rGRs z_Cached.__init__c Os|dd}|j|tg|Ri|}|j|tjg|Ri|}|s7|jr7||jvr7||_|j|St j |g|Ri|}|sc|jrc||jvrc|j| }| D] \}} t ||| qX||_||_||_|jry|sy||_||j|<|S)NZskip_instance_cacheF)pop _tokenize threading get_ident main_threadidentZcachablerHZ_latesttype__call___get_instance_stateitemssetattrZ _fs_token_ storage_argsstorage_options) rIrJrKskipZfs_tokenZfs_token_main_threadobjinstance_stateattr state_valuer:r:r;rUZs$     z_Cached.__call__)r=r>r?r@rGrU __classcell__r:r:rLr;rDBs rDcs"eZdZdZdZdZddddddeedee edfdee d ee ffd d Z e d e d efddZ dededeed ee eeffddZdNdedeed efddZdNdeed dfddZ   dOdededee deed edf ddZdNdedeed dfdd Z !  dPded"e d#ee deed df d$d%Z dQded'e d(e deed eeeeeefff d)d*Z ! !   dRded"e d(e deed ee d#ee f d+d,Zded eeeeeeefffd-d. ZdNded#ee d eeffd/d0 Z  ! ! ! dSded#ee d1e d'e d(e deed eeeeeeeeffffd2d3ZdNd4ed5edeed dfd6d7Z ded e!fd8d9Z"dTded(e deed eeeffd:d;Z#dd?Z%d@dAZ&ded efdBdCZ'e(dfdUfdDdE Z)e*dFdGZ+dHdIZ,dJdKZ-dLdMZ.Z/S)V HfFileSystemaL Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . token (`bool` or `str`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. block_size (`int`, *optional*): Block size for reading and writing files. expand_info (`bool`, *optional*): Whether to expand the information of the files. **storage_options (`dict`, *optional*): Additional options for the filesystem. See [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.__init__). Usage: ```python >>> from huggingface_hub import hffs >>> # List files >>> hffs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> hffs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with hffs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with hffs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` Specify a token for authentication: ```python >>> from huggingface_hub import HfFileSystem >>> hffs = HfFileSystem(token=token) ``` r0ZhfN)endpointtoken block_size expand_inforbrcrdrecsNtj|i||p tj|_||_t||d|_||_||_ i|_ i|_ dS)N)rbrc) rFrGrENDPOINTrbrcr_apirdre_repo_and_revision_exists_cachedircache)r8rbrcrdrerJrZrLr:r;rGs  zHfFileSystem.__init__threading_identr/cs\dptjd<dd<fddtD|||f}tt|}|S)zDeterministic token for cachingrbrcci|]}||qSr:r:).0keyrKr:r; z*HfFileSystem._tokenize..)r4rrfsortedr&rAencode hexdigest)rIrjrJrKZ tokenize_argshr:rnr;rOs  zHfFileSystem._tokenizer(r)r*c Cs|||f|jvrqz |jj|||tjdWnKttfy=}zd|f|j|||f<d|f|j||df<WYd}~n8d}~wty`}zd|f|j|||f<d|j||df<WYd}~nd}~wwd|j|||f<d|j||df<|j|||fS)N)r*r(timeoutF)TN)rhrgZ repo_inforZHF_HUB_ETAG_TIMEOUTrr"r)r8r(r)r*er:r:r;_repo_and_revision_exists"   z%HfFileSystem._repo_and_revision_existpathcCsdttdttdttfdd}||}|std|dddtjvr@d|vr2td|dd \}}tj|}ntj }| ddkrd d |dd d vr|d d \}}d|vrt |}|d ur|d |fvrt d |d}|}n |dd \}}nd }|t||}||||\} } | st|| nd }d |dd d } d |dd d } |dd} d |dd d }| }| }||||\} } | st| ttfr| }|}||||\} }| st|| n1t|| n+|}d }d |vr!|d d \}}|t||}nd }||||\} }| s3td|d ur:|ntj}t|||||dS)a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. revision_in_pathr*r/cSs:|dur|dur||krtd|d|d|S|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)ryr*r:r:r;%_align_revision_in_path_with_revisionszHHfFileSystem.resolve_path.._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r2rrr1Nr0)r.)r rAZ_strip_protocolNotImplementedErrorsplitrr3valuesZREPO_TYPES_MAPPINGZREPO_TYPE_MODELcountjoinSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprrw_raise_file_not_found isinstancerr"r6r')r8rxr*r{r(r)rymatchr+Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r:r:r; resolve_pathsx        zHfFileSystem.resolve_pathcCs|s|j|jdS||}|}|r'|j|d||}|s|jsE|j|j|j dfd|j|j|j |j fddSdS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) riclearrhrr<rN_parentr+r(r)r*)r8rx resolved_pathr:r:r;invalidate_cache>s   zHfFileSystem.invalidate_cacherbmode)HfFileSystemFileHfFileSystemStreamFilecKsj|dur|n|j}|dur||d<d|vrtd|dkr)t||f||d|St||f||d|S)Nrdaz/Appending to remote files is not yet supported.r)rr*)rdr}rr)r8rxrrdr*rKr:r:r;_openZszHfFileSystem._openc KsR|j||d}|jj|j|j|j|j|j|d|dd|j | ddS)Nr*commit_messagecommit_description)r+r)rcr(r*rrrx) rrgZ delete_filer+r)rcr(r*r4rr<)r8rxr*rKrr:r:r;_rmls zHfFileSystem._rmF recursivemaxdepthc sj||d}j||||d}fdd|D}dd|D} d|d} | |r+dnd 7} | |d ur9d |dnd 7} jj|j|jj| |j|d | |d dj | dd S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). > [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. r)rrr*cs"g|] }|s|jqSr:)isdirrr+)rlrxr8r:r; "z#HfFileSystem.rm..cSsg|]}t|dqS))r+)r)rlr+r:r:r;rrpzDelete  z recursively r0Nz up to depth rr)r)r(rc operationsr*rrr) rZ expand_pathrg create_commitr)r(rcr*r4rr<) r8rxrrr*rKrpathsZ paths_in_reporrr:rr;rmys"   zHfFileSystem.rmTdetailrefreshc s|j|d}|z|jf||d|}Wn4tyM|js(td|j|f||d|}fdd|D}t|dkrKtdYnw|rR|Sdd|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). > [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `list[Union[str, dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). r)rr*Ncg|] }|dkr|qSnamer:rlorr:r;rz#HfFileSystem.ls..rcSsg|]}|dqSrr:rr:r:r;rs)rr<_ls_treerr+rrlen)r8rxrrr*rKroutr:rr;lss    zHfFileSystem.lsc s|dur|n jdurjnd}j||d}|}t|j|j|jd|jd}g} |jvr |s j|} | | g} |rdt fdd| D} | r| \} |dus`|kr| djvro| | dnj| d} | | | fd d| D| sRg}|rfd d| D}|r| s|r |r t j| |}|d s||ks|t| |vr|d n||durՈt|dd }||8}fd d| D} tjD]}|d rj|dqjd| j|d |||d| Sjj|j|j|||j|jd}|D]W}|d |j}t|tr>||jd|j |j!|j"|j#|j$d}n |dd|j%|j#d}|d}j&|g ||t|dd |dusn|krs| |q| S)NFrr0r+r.r|cs g|] }|ddkr|fqSrT directoryr:rl path_infodepthr:r;rs z)HfFileSystem._ls_tree..rcs$g|]}|ddkrd|fqS)rTrrr:rrr:r;rs   cs&g|]}|ddur|dqS) last_commitNr)rrrr:r;rs&r2cs"g|] }|dds|qS)rr2) startswithr) common_pathr:r;rrT)rrr*rer)rexpandr*r(filersizerTblob_idlfsxet_hashrsecurityrrrrrTtree_idr)'rerr<r'r(r)r*r.riextendrpopleftappendosrx commonprefixendswithrr5rrrlistrrNrrgZlist_repo_treer+rr!rrrrrrr setdefault)r8rxrrr*rerr root_pathrZcached_path_infosZdirs_not_in_dircacheZ dirs_to_visitdir_infoZdirs_not_expanded common_prefixZcommon_path_depthZ cached_pathtreer cache_pathZcache_path_info parent_pathr:)rrr8r;rs       -   zHfFileSystem._ls_treec/s>|j||dd}tj|g|Ri|EdHdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[tuple[str, list[str], list[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r*rN)rr4r<rFwalk)r8rxrJrKrLr:r;rIs $zHfFileSystem.walkc s0|j||dd}tj|fd|i|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `list[str]`: List of paths matching the pattern. r*rr)rr4r<rFglob)r8rxrrKrLr:r;rYs zHfFileSystem.globwithdirsc s"|dur |dkr td|j||d}|}z|j|fd||j|d|Wn-tyUz|j|fd|i|dd krD|iiniWn tyRiYnwYn+w|s`d d Dn|j|fd|ji|} | dd krw| gnd dDt} |s| Sfdd| DS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[list[str], dict[str, dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rT)rrr*rr*rTrcSsg|] }|ddkr|qSrr:rr:r:r;rrz%HfFileSystem.find..rcSsi|]}|d|qSrr:rr:r:r;rorpz%HfFileSystem.find..crkr:r:)rlrrr:r;rorp) rzrr<rr*rinfoFileNotFoundErrorrq) r8rxrrrrr*rKrrnamesr:rr;findis@      zHfFileSystem.findpath1path2c Ks,|j||d}|j||d}|j|jko|j|jk}|rEd|d|}|jj|j|j|j|d||ddt|j|j|jdgdn?|j |d |jd } | } Wd n1s]wYd|d|}|jj | |j|j|j |j|j|d||dd |j |d |j |d d S) a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rzCopy z to rrr0)Zsrc_path_in_repor+Z src_revision)r)r(r*rrrrNZpath_or_fileobjr+r)rcr(r*rrr)rr(r)rgrr*r4rr+openread upload_filercrr<) r8rrr*rKZresolved_path1Zresolved_path2Z same_reporfcontentr:r:r;cp_filesF     zHfFileSystem.cp_filecKs*|j|fii|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. reTrdater)r8rxrKrr:r:r;modifieds zHfFileSystem.modifiedc s|j|d}||d|jdur|jnd}|jsEdddd|rD|jj|j|j|j dd }idt |j |j |j d d nd|}|sX||jvrX||||jvrtfd d |j|D} | sptd| d|sdus|rrddur|jj|j|j||j |jd} | std| d} t|j|j|j d|jd} t| tr| d| j| jd| j| j| j| j| jdn| d| jdd| j| jd|sfdddDdusJS)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rreNFrr)rrrTr)r(r*)oidtitler)rrcrrr:rrr:r;r%rz%HfFileSystem.info..r)rr*r(r0rr2rrrcrkr:r:)rlkrr:r;roOrpz%HfFileSystem.info..)rrrT)rr<r4rer+rgZlist_repo_commitsr)r(r*r commit_idrZ created_atrrirrZget_paths_infor'r.rr!rxrrrrrrr) r8rxrr*rKrrerrZout1Z paths_inforrr:)rrxr;rs            zHfFileSystem.infocKs<z|ddr |||j|fi|WdSYdS)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFT)r4rr)r8rxrKr:r:r;existsSs  zHfFileSystem.existscCs*z ||ddkWStyYdSw)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rTrF)rOSErrorr8rxr:r:r;rls   zHfFileSystem.isdircCs"z ||ddkWSYdS)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rTrFrrr:r:r;isfile~s zHfFileSystem.isfilecCsB||}t|j|j|j|j|jd}||r|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r(r*rbz /resolve/z/tree/r) rrr)r+r(r*rbrreplace)r8rxrurlr:r:r;rs  zHfFileSystem.urlc sb|d}t|dh}t|ttfrt|dkr)tj||f||d|St |r0|}n| |r>t j |dddSt|t tfrPt j t j|ddd}|dur]t|d}d}|} |j||d } |j||d d } || z2tt| j| j| j| j|jd ||| d|jt|tr|jndd | | W|r|!dSdS|r|!ww) aC Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r*r)callbackoutfileT)exist_okNFwbrrr)r*filenamer(rb)r temp_fileZdisplayed_filename expected_sizeZ resume_sizeheadersZ _tqdm_bar)"r4setkeysrrrrrFget_filerrrmakedirsrAr rxdirnamertellrrZset_sizerrr)r*r+r(rbrg_build_hf_headersZtqdmseekclose) r8ZrpathZlpathrrrKr*Zunhandled_kwargsZ close_fileZ initial_posZresolve_remote_pathrrLr:r;rsR       zHfFileSystem.get_filecCtd)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.r}rr:r:r; transactions zHfFileSystem.transactioncCr)z@Begin write transaction for deferring files, non-context versionrrrr:r:r;start_transactionszHfFileSystem.start_transactioncCstt||j|j|ffSrE) make_instancerTrYrZrVrr:r:r; __reduce__s zHfFileSystem.__reduce__cCst|jt|jdS)N)rirh)rrirhrr:r:r;rVsz HfFileSystem._get_instance_staterE)rNN)FNN)TFN)FFNNN)NFFFN)FNr/N)0r=r>r?r@Z root_markerprotocolr rArboolintrG classmethodrOtuple Exceptionrwr'rrrrrrdictr rrr rrrrrrrrrrrrrpropertyr r r rVr`r:r:rLr;rats0   _  / + ,z$ ?8(cD   ra) metaclasscseZdZddededeeffdd ZfddZd ed ed e fd d Z dddZ dde d dfddZ dfdd Zd efddZZS)rNfsrxr*c stz |j||d|_Wnty'}zd|ddvr"t|d|d}~wwtj||jfi||dS)Nrwrr0B. Make sure the repository and revision exist before writing data.)rrrr4rFrGr<)r8rrxr*rKrvrLr:r;rG szHfFileSystemFile.__init__cst|dsdStSNr)hasattrrF__del__rrLr:r;rs  zHfFileSystemFile.__del__startendr/cCsjdd|d|di|jj}t|jj|jj|jj|jj|jj d}t d||t j d}t ||jS)Nrangebytes=-rrGETrru)rrgrrrr)r*r+r(rbr$rHF_HUB_DOWNLOAD_TIMEOUTr#r)r8rrrrrr:r:r; _fetch_ranges zHfFileSystemFile._fetch_rangecCstjddd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFilerrr:r:r;_initiate_upload/sz!HfFileSystemFile._initiate_uploadFfinalc Cs|jd|j}|j||rN|j|jjj|jj |j j |j j |jj |j j|j j|jd|jddt|jj |jj|j ddSdS)Nrrrrr)bufferrrrwriterrrgrrrr+r)rcr(r*rKr4rremoverr<)r8r,blockr:r:r; _upload_chunk2s(        zHfFileSystemFile._upload_chunkrcs|jdkr;|dus |dkr;|jdkr;|jj|jddd}|}|jt|7_|WdS1s6wYt|S)zRead remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rNrr)rd)rlocrrrxrrrF)r8lengthrrrLr:r;rGs$ zHfFileSystemFile.readcC|j|jSrErrrxrr:r:r;rTzHfFileSystemFile.urlrEr )Fr)r=r>r?rarAr rGrrbytesr&r+rr1rrr`r:r:rLr;r s    rc seZdZ    d!dedededeed ed ef fd d Zd"d edefddZd#defddZ d#de e dede fddZ defddZ fddZddZdd ZZS)$rrNrnonerrxrr*rd cache_typec s|dkr td||dkrtd|d|vr"td|dz |j||d|_Wn"tyN}zd|d d vrDt|d |WYd}~nd}~ww|jdd |_tj||jf|||d |d|_ |t |_ d|_ t |_dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got r9z?HfFileSystemStreamFile only supports cache_type='none' but got rz;HfFileSystemStreamFile only supports reading but got mode=''rrr0r)rrrrdr:)rzrrrr4r<detailsrFrGresponser _exit_stack_stream_iterator bytearray_stream_buffer) r8rrxrr*rdr:rKrvrLr:r;rGYs>    zHfFileSystemStreamFile.__init__r2whencecCs2|dkr |dkr dS||jkr|dkrdStd)NrrzCannot seek streaming HF file)r2rz)r8r2rCr:r:r;r|s zHfFileSystemStreamFile.seekrr3cCs|jdur |d} z |jdus|jdurWdS||j|}|jt|7_|WStyH|jdur=|j|r@|d}Ynwq )zRead the remote file. If the file is already open, we reuse the connection. Otherwise, open a new connection and read from it. If reading the stream fails, we retry with a new connection. NFT)r>_open_connectionr@_read_from_streamr2rrr)r8r3Z retried_oncerr:r:r;rs&    zHfFileSystemStreamFile.readiteratorr/cCs|dkrdS|dkr"t|j}|j|D]}||qt|S|t|jkr:t|jd|}|jd|=|St|j}|j|D]+}|t|}|t|krZ||qF||d||j||dt|St|S)zRead up to `length` bytes from stream buffer and stream. If length < 0, read until EOF. If EOF is reached before length, fewer bytes may be returned. rrDN)rArBrrr8r)r8rGr3bufchunkresultZneedr:r:r;rFs,         z(HfFileSystemStreamFile._read_from_streamcCr4rEr5rr:r:r;rr6zHfFileSystemStreamFile.urlcs"t|dsdS|jtSr)rr?rrFrrrLr:r;rs   zHfFileSystemStreamFile.__del__cCst|j|j|j|j|jjffSrE)reopenrrxr blocksizecacherrr:r:r;r sz!HfFileSystemStreamFile.__reduce__c Cs|jd|_t|jj|jj|jj|jj|j j d}|j j }|j dkr/d|j d|d<|jtd||tjd|_zt|jWntya}z|jjd kr\d|_WYd}~dSd}~ww|j|_dS) z%Open a connection to the remote file.Nrrr r!Ranger"r#i)rBrr@rrr)r*r+r(rrbrgrr2r? enter_contextr%rr$r>r#r status_codeZ iter_bytes)r8rrrvr:r:r;rEs<     z'HfFileSystemStreamFile._open_connection)rNrr9)rr7)r=r>r?rarAr rrGrrr r8rFrrr rEr`r:r:rLr;rXs2#" rr*r/cCst|r|St|SrE)rr safe_quoterr:r:r;r7sr7scCs t|ddS)Nr0)safe)r)rRr:r:r;rQs rQrxrcCsN|}t|tr |d}nt|tr|d}n t|tr"|d}t||)Nz (repository not found)z (revision not found)z (invalid repository id))rrrr"r)rxrmsgr:r:r;rs       rrrrdr:cCs|j||||dS)Nr<)r)rrxrrdr:r:r:r;rKsrKcCs0||i|}|D] \}}t|||q |SrE)rWrX)rIrJrKr]rr^r_r:r:r;r  sr )Rrrer)rP collectionsr contextlibrcopyr dataclassesrrr itertoolsrpathlibr typingr r r r r urllib.parserrZfsspecZhttpxZfsspec.callbacksrrrZ fsspec.utilsrr0rZ _commit_apirrerrorsrrrrZ file_downloadrrZhf_apirr r!utilsr"r#r$r%Zutils.insecure_hashlibr&compileVERBOSErr'rTZAbstractFileSystemrCrBrDraspecZAbstractBufferedFilerrrAr7rQrrrrKr Zhffsr:r:r:r;sb           2L