o
    1 i                   	   @   s   d dl Z d dlZd dlZd dlmZmZmZmZmZ d dl	m
Z
mZ er(d dlZdedeee  defddZ	dd	eeee f d
ed deee df fddZdddefddZdd ZdefddZdefddZdedefddZdd Zdd ZdS )    N)TYPE_CHECKINGListOptionalTupleUnion)RetryingPyFileSystem_resolve_custom_schemepath
extensionsreturnc                    sN   |du st |tsJ t||du rdS dd |D }t fdd|D S )a  Check if a path has a file extension in the provided list.

    Examples:
        >>> _has_file_extension("foo.csv", ["csv"])
        True
        >>> _has_file_extension("foo.CSV", ["csv"])
        True
        >>> _has_file_extension("foo.csv", ["json", "jsonl"])
        False
        >>> _has_file_extension("foo.csv", None)
        True

    Args:
        path: The path to check.
        extensions: A list of extensions to check against. If `None`, any extension is
            considered valid.
    NTc                 S   s   g | ]	}d |   qS ).)lower.0ext r   i/home/app/PaddleOCR-VL-test/.venv_paddleocr/lib/python3.10/site-packages/ray/data/datasource/path_util.py
<listcomp>$   s    z'_has_file_extension.<locals>.<listcomp>c                 3   s    | ]
}   |V  qd S N)r   endswithr   r	   r   r   	<genexpr>%   s    z&_has_file_extension.<locals>.<genexpr>)
isinstancelisttypeany)r	   r
   r   r   r   _has_file_extension   s
   r   paths
filesystemzpyarrow.fs.FileSystemc                 C   s  ddl }ddlm}m}m}m} t| tr| g} t| tj	r$t| g} n t| t
r2tdd | D r:td|  dt| dkrDtd|r{t||s{d	| }zddl}dd
lm}	 W n tyh   t|dw t||jjsut|d|||}g }
| D ]}t|}z	|||\}}W n` |jjy } zRdt|v r|t||\}}t|}n8dt|v rtjj|ddj}|dv rzddl}dd
lm}	 W n ty   tddw |||	 }|}n  W Y d}~nd}~ww |du r|}t|st |}|!|}|
"| q|
|fS )at  
    Resolves and normalizes all provided paths, infers a filesystem from the
    paths and ensures that all paths use the same filesystem.

    Args:
        paths: A single file/directory path or a list of file/directory paths.
            A list of paths can contain both files and directories.
        filesystem: The filesystem implementation that should be used for
            reading these files. If None, a filesystem will be inferred. If not
            None, the provided filesystem will still be validated against all
            filesystems inferred from the provided paths to ensure
            compatibility.
    r   N)
FileSystemFSSpecHandlerPyFileSystem_resolve_filesystem_and_pathc                 s   s    | ]	}t |t V  qd S r   )r   str)r   pr   r   r   r   E   s    z0_resolve_paths_and_filesystem.<locals>.<genexpr>zIExpected `paths` to be a `str`, `pathlib.Path`, or `list[str]`, but got ``zMust provide at least one path.zThe filesystem passed must either conform to pyarrow.fs.FileSystem, or fsspec.spec.AbstractFileSystem. The provided filesystem was: HTTPFileSystemzCannot parse URIz#Unrecognized filesystem type in URIFallow_fragmentshttphttpsz.Please install fsspec to read files from HTTP.)#pyarrow
pyarrow.fsr   r    r!   r"   r   r#   pathlibPathr   r   
ValueErrorlenfsspecfsspec.implementations.httpr'   ModuleNotFoundError	TypeErrorspecZAbstractFileSystemr   libZArrowInvalid_encode_url_decode_urlurllibparseurlparseschemeImportError_is_http_filesystem_unwrap_protocolnormalize_pathappend)r   r   par   r    r!   r"   err_msgr3   r'   resolved_pathsr	   Zresolved_filesystemresolved_pather>   r   r   r   _resolve_paths_and_filesystem(   s   




rI   fsc                 C   sx   ddl m}m} zddl}ddlm} W n
 ty   Y dS w t| tr(| 	 } t| |s/dS t| j
|o;t| j
j|S )zKReturn whether ``fs`` is a PyFileSystem handled by a fsspec HTTPFileSystem.r   )r    r!   Nr&   F)r.   r    r!   r3   r4   r'   r5   r   r   unwraphandlerrJ   )rJ   r    r!   r3   r'   r   r   r   r@      s   

r@   c                 C   s   t jdkrt| rt|  S tjj| dd}|j	r d|j	 nd}|j
r*d|j
 nd}|j}|jdkrAd|jv rA|jdd	 }|j}t jdkrk|skt|d
krk|d dkrk|d  rk|dd dv rk|dd }|| | | S )z2
    Slice off any protocol prefixes on path.
    win32Fr(   ; ?Zs3@   r   /         ):z:/N)sysplatform_is_local_windows_pathr/   r0   as_posixr;   r<   r=   paramsquerynetlocr>   splitr	   r2   isalpha)r	   parsedr]   r^   r_   Zparsed_pathr   r   r   rA      s&   

rA   c                 C   s   t j| jdkS )NrO   r;   r<   r=   r>   r   r   r   r   _is_url   s   rd   c                 C   s   t j| }|jdv S )Nr*   rc   )r	   rb   r   r   r   _is_http_url   s   
re   c                 C   sn   t jdkrdS t| dkr| d dkrdS t| dkr5| d dkr5| d	 d
ks-| d	 dkr5| d  r5dS dS )z5Determines if path is a Windows file-system location.rM   FrU   r   \TrS   rX   rV   rT   )rY   rZ   r2   ra   r   r   r   r   r[      s   

r[   c                 C   s   t jj| ddS )Nz/:)safe)r;   r<   quoter   r   r   r   r9      s   r9   c                 C   s   t j| S r   )r;   r<   unquoter   r   r   r   r:      s   r:   r   )r/   rY   r;   typingr   r   r   r   r   Zray.data._internal.utilr   r   r-   r#   boolr   rI   r@   rA   rd   re   r[   r9   r:   r   r   r   r   <module>   s.    
k"